From b2e92b1d100e7bec198d82a978200e80bbb2037c Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 5 Jul 2019 19:14:23 +0700 Subject: [PATCH 001/114] Import the redesigned AnnounceShare under the new name TrackerShare --- plugins/TrackerShare/Test/TestTrackerShare.py | 24 ++ plugins/TrackerShare/Test/conftest.py | 3 + plugins/TrackerShare/Test/pytest.ini | 5 + plugins/TrackerShare/TrackerSharePlugin.py | 363 ++++++++++++++++++ plugins/TrackerShare/__init__.py | 1 + 5 files changed, 396 insertions(+) create mode 100644 plugins/TrackerShare/Test/TestTrackerShare.py create mode 100644 plugins/TrackerShare/Test/conftest.py create mode 100644 plugins/TrackerShare/Test/pytest.ini create mode 100644 plugins/TrackerShare/TrackerSharePlugin.py create mode 100644 plugins/TrackerShare/__init__.py diff --git a/plugins/TrackerShare/Test/TestTrackerShare.py b/plugins/TrackerShare/Test/TestTrackerShare.py new file mode 100644 index 00000000..391f2009 --- /dev/null +++ b/plugins/TrackerShare/Test/TestTrackerShare.py @@ -0,0 +1,24 @@ +import pytest + +from TrackerShare import TrackerSharePlugin +from Peer import Peer +from Config import config + + +@pytest.mark.usefixtures("resetSettings") +@pytest.mark.usefixtures("resetTempSettings") +class TestTrackerShare: + def testAnnounceList(self, file_server): + open("%s/trackers.json" % config.data_dir, "w").write("{}") + tracker_storage = TrackerSharePlugin.tracker_storage + tracker_storage.load() + peer = Peer(file_server.ip, 1544, connection_server=file_server) + assert peer.request("getTrackers")["trackers"] == [] + + tracker_storage.onTrackerFound("zero://%s:15441" % file_server.ip) + assert peer.request("getTrackers")["trackers"] == [] + + # It needs to have at least one successfull announce to be shared to other peers + tracker_storage.onTrackerSuccess("zero://%s:15441" % file_server.ip, 1.0) + assert peer.request("getTrackers")["trackers"] == ["zero://%s:15441" % file_server.ip] + diff --git a/plugins/TrackerShare/Test/conftest.py b/plugins/TrackerShare/Test/conftest.py new file mode 100644 index 00000000..5abd4dd6 --- /dev/null +++ b/plugins/TrackerShare/Test/conftest.py @@ -0,0 +1,3 @@ +from src.Test.conftest import * + +from Config import config diff --git a/plugins/TrackerShare/Test/pytest.ini b/plugins/TrackerShare/Test/pytest.ini new file mode 100644 index 00000000..d09210d1 --- /dev/null +++ b/plugins/TrackerShare/Test/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +python_files = Test*.py +addopts = -rsxX -v --durations=6 +markers = + webtest: mark a test as a webtest. \ No newline at end of file diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py new file mode 100644 index 00000000..244b0be5 --- /dev/null +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -0,0 +1,363 @@ +import random +import time +import os +import logging +import json +import atexit +import re + +import gevent + +from Config import config +from Plugin import PluginManager +from util import helper + + +class TrackerStorage(object): + def __init__(self): + self.site_announcer = None + self.log = logging.getLogger("TrackerStorage") + + self.working_tracker_time_interval = 60 * 60 + self.tracker_down_time_interval = 60 * 60 + self.tracker_discover_time_interval = 5 * 60 + + self.working_shared_trackers_limit_per_protocol = {} + self.working_shared_trackers_limit_per_protocol["other"] = 2 + + self.file_path = "%s/trackers.json" % config.data_dir + self.load() + self.time_discover = 0.0 + self.time_success = 0.0 + atexit.register(self.save) + + def initTrackerLimitForProtocol(self): + for s in re.split("[,;]", config.working_shared_trackers_limit_per_protocol): + x = s.split("=", 1) + if len(x) == 1: + x = ["other", x[0]] + try: + self.working_shared_trackers_limit_per_protocol[x[0]] = int(x[1]) + except ValueError: + pass + self.log.debug("Limits per protocol: %s" % self.working_shared_trackers_limit_per_protocol) + + def getTrackerLimitForProtocol(self, protocol): + l = self.working_shared_trackers_limit_per_protocol + return l.get(protocol, l.get("other")) + + def setSiteAnnouncer(self, site_announcer): + if self.site_announcer: + return + self.site_announcer = site_announcer + self.initTrackerLimitForProtocol() + self.recheckValidTrackers() + + def isTrackerAddressValid(self, tracker_address): + if not self.site_announcer: # Not completely initialized, skip check + return True + + address_parts = self.site_announcer.getAddressParts(tracker_address) + if not address_parts: + self.log.debug("Invalid tracker address: %s" % tracker_address) + return False + + handler = self.site_announcer.getTrackerHandler(address_parts["protocol"]) + if not handler: + self.log.debug("Invalid tracker address: Unknown protocol %s: %s" % (address_parts["protocol"], tracker_address)) + return False + + return True + + def recheckValidTrackers(self): + trackers = self.getTrackers() + for address, tracker in list(trackers.items()): + if not self.isTrackerAddressValid(address): + del trackers[address] + + def getNormalizedTrackerProtocol(self, tracker_address): + if not self.site_announcer: + return None + + address_parts = self.site_announcer.getAddressParts(tracker_address) + if not address_parts: + return None + + protocol = address_parts["protocol"] + if protocol == "https": + protocol = "http" + + return protocol + + def getSupportedProtocols(self): + if not self.site_announcer: + return None + + supported_trackers = self.site_announcer.getSupportedTrackers() + + # If a tracker is in our list, but is absent from the results of getSupportedTrackers(), + # it seems to be supported by software, but forbidden by the settings or network configuration. + # We check and remove thoose trackers here, since onTrackerError() is never emitted for them. + trackers = self.getTrackers() + for tracker_address, tracker in list(trackers.items()): + t = max(trackers[tracker_address]["time_added"], + trackers[tracker_address]["time_success"]) + if tracker_address not in supported_trackers and t < time.time() - self.tracker_down_time_interval: + self.log.debug("Tracker %s looks unused, removing." % tracker_address) + del trackers[tracker_address] + + protocols = set() + for tracker_address in supported_trackers: + protocol = self.getNormalizedTrackerProtocol(tracker_address) + if not protocol: + continue + protocols.add(protocol) + + protocols = list(protocols) + + self.log.debug("Supported tracker protocols: %s" % protocols) + + return protocols + + def getDefaultFile(self): + return {"shared": {}} + + def onTrackerFound(self, tracker_address, type="shared", my=False): + if not self.isTrackerAddressValid(tracker_address): + return False + + trackers = self.getTrackers(type) + added = False + if tracker_address not in trackers: + trackers[tracker_address] = { + "time_added": time.time(), + "time_success": 0, + "latency": 99.0, + "num_error": 0, + "my": False + } + self.log.debug("New tracker found: %s" % tracker_address) + added = True + + trackers[tracker_address]["time_found"] = time.time() + trackers[tracker_address]["my"] = my + return added + + def onTrackerSuccess(self, tracker_address, latency): + trackers = self.getTrackers() + if tracker_address not in trackers: + return False + + trackers[tracker_address]["latency"] = latency + trackers[tracker_address]["time_success"] = time.time() + trackers[tracker_address]["num_error"] = 0 + + self.time_success = time.time() + + def onTrackerError(self, tracker_address): + trackers = self.getTrackers() + if tracker_address not in trackers: + return False + + trackers[tracker_address]["time_error"] = time.time() + trackers[tracker_address]["num_error"] += 1 + + if self.time_success < time.time() - self.tracker_down_time_interval / 2: + # Don't drop trackers from the list, if there haven't been any successful announces recently. + # There may be network connectivity issues. + return + + protocol = self.getNormalizedTrackerProtocol(tracker_address) or "" + + nr_working_trackers_for_protocol = len(self.getTrackersPerProtocol(working_only=True).get(protocol, [])) + nr_working_trackers = len(self.getWorkingTrackers()) + + error_limit = 30 + if nr_working_trackers_for_protocol >= self.getTrackerLimitForProtocol(protocol): + error_limit = 10 + if nr_working_trackers >= config.working_shared_trackers_limit: + error_limit = 5 + + if trackers[tracker_address]["num_error"] > error_limit and trackers[tracker_address]["time_success"] < time.time() - self.tracker_down_time_interval: + self.log.debug("Tracker %s looks down, removing." % tracker_address) + del trackers[tracker_address] + + def isTrackerWorking(self, tracker_address, type="shared"): + trackers = self.getTrackers(type) + tracker = trackers[tracker_address] + if not tracker: + return False + + if tracker["time_success"] > time.time() - self.working_tracker_time_interval: + return True + + return False + + def getTrackers(self, type="shared"): + return self.file_content.setdefault(type, {}) + + def getTrackersPerProtocol(self, type="shared", working_only=False): + if not self.site_announcer: + return None + + trackers = self.getTrackers(type) + + trackers_per_protocol = {} + for tracker_address in trackers: + protocol = self.getNormalizedTrackerProtocol(tracker_address) + if not protocol: + continue + if not working_only or self.isTrackerWorking(tracker_address, type=type): + trackers_per_protocol.setdefault(protocol, []).append(tracker_address) + + return trackers_per_protocol + + def getWorkingTrackers(self, type="shared"): + trackers = { + key: tracker for key, tracker in self.getTrackers(type).items() + if self.isTrackerWorking(key, type) + } + return trackers + + def getFileContent(self): + if not os.path.isfile(self.file_path): + open(self.file_path, "w").write("{}") + return self.getDefaultFile() + try: + return json.load(open(self.file_path)) + except Exception as err: + self.log.error("Error loading trackers list: %s" % err) + return self.getDefaultFile() + + def load(self): + self.file_content = self.getFileContent() + + trackers = self.getTrackers() + self.log.debug("Loaded %s shared trackers" % len(trackers)) + for address, tracker in list(trackers.items()): + tracker["num_error"] = 0 + self.recheckValidTrackers() + + def save(self): + s = time.time() + helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True).encode("utf8")) + self.log.debug("Saved in %.3fs" % (time.time() - s)) + + def enoughWorkingTrackers(self, type="shared"): + supported_protocols = self.getSupportedProtocols() + if not supported_protocols: + return False + + trackers_per_protocol = self.getTrackersPerProtocol(type="shared", working_only=True) + if not trackers_per_protocol: + return False + + total_nr = 0 + + for protocol in supported_protocols: + trackers = trackers_per_protocol.get(protocol, []) + if len(trackers) < self.getTrackerLimitForProtocol(protocol): + self.log.debug("Not enough working trackers for protocol %s: %s < %s" % ( + protocol, len(trackers), self.getTrackerLimitForProtocol(protocol))) + return False + total_nr += len(trackers) + + if total_nr < config.working_shared_trackers_limit: + self.log.debug("Not enough working trackers: %s < %s" % ( + total_nr, config.working_shared_trackers_limit)) + return False + + return True + + def discoverTrackers(self, peers): + if self.enoughWorkingTrackers(type="shared"): + return False + + self.log.debug("Discovering trackers from %s peers..." % len(peers)) + + s = time.time() + num_success = 0 + for peer in peers: + if peer.connection and peer.connection.handshake.get("rev", 0) < 3560: + continue # Not supported + + res = peer.request("getTrackers") + if not res or "error" in res: + continue + + num_success += 1 + + random.shuffle(res["trackers"]) + for tracker_address in res["trackers"]: + if type(tracker_address) is bytes: # Backward compatibilitys + tracker_address = tracker_address.decode("utf8") + added = self.onTrackerFound(tracker_address) + if added: # Only add one tracker from one source + break + + if not num_success and len(peers) < 20: + self.time_discover = 0.0 + + if num_success: + self.save() + + self.log.debug("Trackers discovered from %s/%s peers in %.3fs" % (num_success, len(peers), time.time() - s)) + + +if "tracker_storage" not in locals(): + tracker_storage = TrackerStorage() + + +@PluginManager.registerTo("SiteAnnouncer") +class SiteAnnouncerPlugin(object): + def getTrackers(self): + tracker_storage.setSiteAnnouncer(self) + if tracker_storage.time_discover < time.time() - tracker_storage.tracker_discover_time_interval: + tracker_storage.time_discover = time.time() + gevent.spawn(tracker_storage.discoverTrackers, self.site.getConnectedPeers()) + trackers = super(SiteAnnouncerPlugin, self).getTrackers() + shared_trackers = list(tracker_storage.getTrackers("shared").keys()) + if shared_trackers: + return trackers + shared_trackers + else: + return trackers + + def announceTracker(self, tracker, *args, **kwargs): + tracker_storage.setSiteAnnouncer(self) + res = super(SiteAnnouncerPlugin, self).announceTracker(tracker, *args, **kwargs) + if res: + latency = res + tracker_storage.onTrackerSuccess(tracker, latency) + elif res is False: + tracker_storage.onTrackerError(tracker) + + return res + + +@PluginManager.registerTo("FileRequest") +class FileRequestPlugin(object): + def actionGetTrackers(self, params): + shared_trackers = list(tracker_storage.getWorkingTrackers("shared").keys()) + random.shuffle(shared_trackers) + self.response({"trackers": shared_trackers}) + + +@PluginManager.registerTo("FileServer") +class FileServerPlugin(object): + def portCheck(self, *args, **kwargs): + res = super(FileServerPlugin, self).portCheck(*args, **kwargs) + if res and not config.tor == "always" and "Bootstrapper" in PluginManager.plugin_manager.plugin_names: + for ip in self.ip_external_list: + my_tracker_address = "zero://%s:%s" % (ip, config.fileserver_port) + tracker_storage.onTrackerFound(my_tracker_address, my=True) + return res + + +@PluginManager.registerTo("ConfigPlugin") +class ConfigPlugin(object): + def createArguments(self): + group = self.parser.add_argument_group("TrackerShare plugin") + group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached (total)', default=10, type=int, metavar='limit') + group.add_argument('--working_shared_trackers_limit_per_protocol', help='Stop discovering new shared trackers after this number of shared trackers reached per each supported protocol', default="zero=5,other=2", metavar='limit') + + return super(ConfigPlugin, self).createArguments() diff --git a/plugins/TrackerShare/__init__.py b/plugins/TrackerShare/__init__.py new file mode 100644 index 00000000..0cc19c27 --- /dev/null +++ b/plugins/TrackerShare/__init__.py @@ -0,0 +1 @@ +from . import TrackerSharePlugin From 3910338b28b1ad300c4f6bdfab55a2feb186a7c5 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 5 Jul 2019 19:16:25 +0700 Subject: [PATCH 002/114] Import plugin: TrackerList --- .../disabled-TrackerList/TrackerListPlugin.py | 107 ++++++++++++++++++ plugins/disabled-TrackerList/__init__.py | 1 + 2 files changed, 108 insertions(+) create mode 100644 plugins/disabled-TrackerList/TrackerListPlugin.py create mode 100644 plugins/disabled-TrackerList/__init__.py diff --git a/plugins/disabled-TrackerList/TrackerListPlugin.py b/plugins/disabled-TrackerList/TrackerListPlugin.py new file mode 100644 index 00000000..c1ced5e1 --- /dev/null +++ b/plugins/disabled-TrackerList/TrackerListPlugin.py @@ -0,0 +1,107 @@ +import time +import os +import logging +import json +import atexit +import re + +import gevent + +from Config import config +from Debug import Debug +from Plugin import PluginManager +from util import helper + +class TrackerList(object): + def __init__(self): + self.log = logging.getLogger("TrackerList") + self.tracker_storage = None + self.last_rescan_time = 0.0 + self.last_rescan_failed = False + + def parse_list(self, data): + for line in data.splitlines(): + line = line.strip() + + if not line: + continue + + if re.match("^udp://", line): + line = re.sub("/announce$", "", line) + + if self.tracker_storage.onTrackerFound(line): + self.log.info("Added tracker: %s" % line) + + def do_rescan(self): + url = config.tracker_list_url + response = None + + self.log.info("Rescanning: %s" % url) + + try: + # FIXME: add support of reading from ZeroNet URLs + if re.match("^http(s)?://", url): + req = helper.httpRequest(url) + response = req.read().decode("utf8") + req.close() + req = None + else: + response = open(url, 'r').read().decode("utf8") + except Exception as err: + self.log.error("Error reading %s: %s" % (url, err)) + self.last_rescan_failed = True + + if response: + self.parse_list(response); + self.last_rescan_failed = False + + def reload(self): + if "AnnounceShare" not in PluginManager.plugin_manager.plugin_names: + return + + rescan_interval = config.tracker_list_rescan_interval + if self.last_rescan_failed: + rescan_interval = (rescan_interval / 2, 60) + + if self.last_rescan_time > time.time() - rescan_interval: + return + + self.last_rescan_time = time.time() + + try: + if "tracker_storage" not in locals(): + from AnnounceShare.AnnounceSharePlugin import tracker_storage + self.tracker_storage = tracker_storage + if self.tracker_storage: + gevent.spawn(self.do_rescan) + except Exception as err: + self.log.error("%s" % Debug.formatException(err)) + + +if "tracker_list" not in locals(): + tracker_list = TrackerList() + + +@PluginManager.registerTo("SiteAnnouncer") +class SiteAnnouncerPlugin(object): + def announceTracker(self, tracker, *args, **kwargs): + tracker_list.reload() + return super(SiteAnnouncerPlugin, self).announceTracker(tracker, *args, **kwargs) + + +@PluginManager.registerTo("FileServer") +class FileServerPlugin(object): + def portCheck(self, *args, **kwargs): + res = super(FileServerPlugin, self).portCheck(*args, **kwargs) + tracker_list.reload() + return res + + +@PluginManager.registerTo("ConfigPlugin") +class ConfigPlugin(object): + def createArguments(self): + group = self.parser.add_argument_group("TrackerList plugin") + group.add_argument('--tracker_list_url', help='URL of local file path, where the list of additional trackers is located', default='https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt', metavar='url') + group.add_argument('--tracker_list_rescan_interval', help='Interval in seconds between rescans of the list of additional trackers', default=60 * 60, type=int, metavar='interval') + + return super(ConfigPlugin, self).createArguments() diff --git a/plugins/disabled-TrackerList/__init__.py b/plugins/disabled-TrackerList/__init__.py new file mode 100644 index 00000000..d834cf6b --- /dev/null +++ b/plugins/disabled-TrackerList/__init__.py @@ -0,0 +1 @@ +from . import TrackerListPlugin From 84526a6657021db0c35b50e581a4978f70bebfca Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 5 Jul 2019 19:28:03 +0700 Subject: [PATCH 003/114] TrackerShare: raise the default limit per protocol limit from 2 to 4 --- plugins/TrackerShare/TrackerSharePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 244b0be5..45b4aea1 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -358,6 +358,6 @@ class ConfigPlugin(object): def createArguments(self): group = self.parser.add_argument_group("TrackerShare plugin") group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached (total)', default=10, type=int, metavar='limit') - group.add_argument('--working_shared_trackers_limit_per_protocol', help='Stop discovering new shared trackers after this number of shared trackers reached per each supported protocol', default="zero=5,other=2", metavar='limit') + group.add_argument('--working_shared_trackers_limit_per_protocol', help='Stop discovering new shared trackers after this number of shared trackers reached per each supported protocol', default="zero=5,other=4", metavar='limit') return super(ConfigPlugin, self).createArguments() From 33c81a89e99991c783a7e870f15f73a16ac45b4a Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 5 Jul 2019 19:38:00 +0700 Subject: [PATCH 004/114] TrackerShare: rename the config arguments to avoid the name clash with AnnounceShare's arguments --working_shared_trackers_limit -> --shared_trackers_limit --working_shared_trackers_limit_per_protocol -> --shared_trackers_limit_per_protocol Also modify the help messages so that they were more consistent with how the code really works. --- plugins/TrackerShare/TrackerSharePlugin.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 45b4aea1..6b32c4e2 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -22,8 +22,8 @@ class TrackerStorage(object): self.tracker_down_time_interval = 60 * 60 self.tracker_discover_time_interval = 5 * 60 - self.working_shared_trackers_limit_per_protocol = {} - self.working_shared_trackers_limit_per_protocol["other"] = 2 + self.shared_trackers_limit_per_protocol = {} + self.shared_trackers_limit_per_protocol["other"] = 2 self.file_path = "%s/trackers.json" % config.data_dir self.load() @@ -32,18 +32,18 @@ class TrackerStorage(object): atexit.register(self.save) def initTrackerLimitForProtocol(self): - for s in re.split("[,;]", config.working_shared_trackers_limit_per_protocol): + for s in re.split("[,;]", config.shared_trackers_limit_per_protocol): x = s.split("=", 1) if len(x) == 1: x = ["other", x[0]] try: - self.working_shared_trackers_limit_per_protocol[x[0]] = int(x[1]) + self.shared_trackers_limit_per_protocol[x[0]] = int(x[1]) except ValueError: pass - self.log.debug("Limits per protocol: %s" % self.working_shared_trackers_limit_per_protocol) + self.log.debug("Limits per protocol: %s" % self.shared_trackers_limit_per_protocol) def getTrackerLimitForProtocol(self, protocol): - l = self.working_shared_trackers_limit_per_protocol + l = self.shared_trackers_limit_per_protocol return l.get(protocol, l.get("other")) def setSiteAnnouncer(self, site_announcer): @@ -175,7 +175,7 @@ class TrackerStorage(object): error_limit = 30 if nr_working_trackers_for_protocol >= self.getTrackerLimitForProtocol(protocol): error_limit = 10 - if nr_working_trackers >= config.working_shared_trackers_limit: + if nr_working_trackers >= config.shared_trackers_limit: error_limit = 5 if trackers[tracker_address]["num_error"] > error_limit and trackers[tracker_address]["time_success"] < time.time() - self.tracker_down_time_interval: @@ -262,9 +262,9 @@ class TrackerStorage(object): return False total_nr += len(trackers) - if total_nr < config.working_shared_trackers_limit: + if total_nr < config.shared_trackers_limit: self.log.debug("Not enough working trackers: %s < %s" % ( - total_nr, config.working_shared_trackers_limit)) + total_nr, config.shared_trackers_limit)) return False return True @@ -357,7 +357,7 @@ class FileServerPlugin(object): class ConfigPlugin(object): def createArguments(self): group = self.parser.add_argument_group("TrackerShare plugin") - group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached (total)', default=10, type=int, metavar='limit') - group.add_argument('--working_shared_trackers_limit_per_protocol', help='Stop discovering new shared trackers after this number of shared trackers reached per each supported protocol', default="zero=5,other=4", metavar='limit') + group.add_argument('--shared_trackers_limit', help='Discover new shared trackers if this number of shared trackers isn\'t reached (total)', default=10, type=int, metavar='limit') + group.add_argument('--shared_trackers_limit_per_protocol', help='Discover new shared trackers if this number of shared trackers isn\'t reached per each supported protocol', default="zero=5,other=4", metavar='limit') return super(ConfigPlugin, self).createArguments() From 8f8e10a703f5d1030468ffcba5ad20564b6182d0 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 5 Jul 2019 23:17:13 +0700 Subject: [PATCH 005/114] TrackerShare: Change the log level for several messages from debug to info Increased the log level for messages that are not very annoying and help to keep the track of events. --- plugins/TrackerShare/TrackerSharePlugin.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 6b32c4e2..3a09c5d2 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -40,7 +40,7 @@ class TrackerStorage(object): self.shared_trackers_limit_per_protocol[x[0]] = int(x[1]) except ValueError: pass - self.log.debug("Limits per protocol: %s" % self.shared_trackers_limit_per_protocol) + self.log.info("Limits per protocol: %s" % self.shared_trackers_limit_per_protocol) def getTrackerLimitForProtocol(self, protocol): l = self.shared_trackers_limit_per_protocol @@ -103,7 +103,7 @@ class TrackerStorage(object): t = max(trackers[tracker_address]["time_added"], trackers[tracker_address]["time_success"]) if tracker_address not in supported_trackers and t < time.time() - self.tracker_down_time_interval: - self.log.debug("Tracker %s looks unused, removing." % tracker_address) + self.log.info("Tracker %s looks unused, removing." % tracker_address) del trackers[tracker_address] protocols = set() @@ -136,7 +136,7 @@ class TrackerStorage(object): "num_error": 0, "my": False } - self.log.debug("New tracker found: %s" % tracker_address) + self.log.info("New tracker found: %s" % tracker_address) added = True trackers[tracker_address]["time_found"] = time.time() @@ -179,7 +179,7 @@ class TrackerStorage(object): error_limit = 5 if trackers[tracker_address]["num_error"] > error_limit and trackers[tracker_address]["time_success"] < time.time() - self.tracker_down_time_interval: - self.log.debug("Tracker %s looks down, removing." % tracker_address) + self.log.info("Tracker %s looks down, removing." % tracker_address) del trackers[tracker_address] def isTrackerWorking(self, tracker_address, type="shared"): @@ -257,13 +257,13 @@ class TrackerStorage(object): for protocol in supported_protocols: trackers = trackers_per_protocol.get(protocol, []) if len(trackers) < self.getTrackerLimitForProtocol(protocol): - self.log.debug("Not enough working trackers for protocol %s: %s < %s" % ( + self.log.info("Not enough working trackers for protocol %s: %s < %s" % ( protocol, len(trackers), self.getTrackerLimitForProtocol(protocol))) return False total_nr += len(trackers) if total_nr < config.shared_trackers_limit: - self.log.debug("Not enough working trackers: %s < %s" % ( + self.log.info("Not enough working trackers: %s < %s" % ( total_nr, config.shared_trackers_limit)) return False @@ -273,7 +273,7 @@ class TrackerStorage(object): if self.enoughWorkingTrackers(type="shared"): return False - self.log.debug("Discovering trackers from %s peers..." % len(peers)) + self.log.info("Discovering trackers from %s peers..." % len(peers)) s = time.time() num_success = 0 @@ -301,7 +301,7 @@ class TrackerStorage(object): if num_success: self.save() - self.log.debug("Trackers discovered from %s/%s peers in %.3fs" % (num_success, len(peers), time.time() - s)) + self.log.info("Trackers discovered from %s/%s peers in %.3fs" % (num_success, len(peers), time.time() - s)) if "tracker_storage" not in locals(): From c8545ce054efbe1189a9ad816e8d043e5f9e11c2 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 5 Jul 2019 23:41:23 +0700 Subject: [PATCH 006/114] TrackerShare: print the total number of discovered trackers at the end of discovery procedure --- plugins/TrackerShare/TrackerSharePlugin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 3a09c5d2..c89723fe 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -277,6 +277,7 @@ class TrackerStorage(object): s = time.time() num_success = 0 + num_trackers_discovered = 0 for peer in peers: if peer.connection and peer.connection.handshake.get("rev", 0) < 3560: continue # Not supported @@ -293,6 +294,7 @@ class TrackerStorage(object): tracker_address = tracker_address.decode("utf8") added = self.onTrackerFound(tracker_address) if added: # Only add one tracker from one source + num_trackers_discovered += 1 break if not num_success and len(peers) < 20: @@ -301,7 +303,7 @@ class TrackerStorage(object): if num_success: self.save() - self.log.info("Trackers discovered from %s/%s peers in %.3fs" % (num_success, len(peers), time.time() - s)) + self.log.info("Discovered %s new trackers from %s/%s peers in %.3fs" % (num_trackers_discovered, num_success, len(peers), time.time() - s)) if "tracker_storage" not in locals(): From d35a15d674ee1b25cfaf909ae48e64705fcd08bc Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 6 Jul 2019 01:15:37 +0700 Subject: [PATCH 007/114] TrackerShare: don't delete "my" trackers on errors, but delete them on program restart; add "persistent" flag for manually added trackers --- plugins/TrackerShare/TrackerSharePlugin.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index c89723fe..e683dbfc 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -122,25 +122,31 @@ class TrackerStorage(object): def getDefaultFile(self): return {"shared": {}} - def onTrackerFound(self, tracker_address, type="shared", my=False): + def onTrackerFound(self, tracker_address, type="shared", my=False, persistent=False): if not self.isTrackerAddressValid(tracker_address): return False trackers = self.getTrackers(type) added = False if tracker_address not in trackers: + # "My" trackers never get deleted on announce errors, but aren't saved between restarts. + # They are to be used as automatically added addresses from the Bootstrap plugin. + # Persistent trackers never get deleted. + # They are to be used for entries manually added by the user. trackers[tracker_address] = { "time_added": time.time(), "time_success": 0, "latency": 99.0, "num_error": 0, - "my": False + "my": False, + "persistent": False } self.log.info("New tracker found: %s" % tracker_address) added = True trackers[tracker_address]["time_found"] = time.time() - trackers[tracker_address]["my"] = my + trackers[tracker_address]["my"] |= my + trackers[tracker_address]["persistent"] |= persistent return added def onTrackerSuccess(self, tracker_address, latency): @@ -162,6 +168,9 @@ class TrackerStorage(object): trackers[tracker_address]["time_error"] = time.time() trackers[tracker_address]["num_error"] += 1 + if trackers[tracker_address]["my"] or trackers[tracker_address]["persistent"]: + return + if self.time_success < time.time() - self.tracker_down_time_interval / 2: # Don't drop trackers from the list, if there haven't been any successful announces recently. # There may be network connectivity issues. @@ -235,7 +244,14 @@ class TrackerStorage(object): trackers = self.getTrackers() self.log.debug("Loaded %s shared trackers" % len(trackers)) for address, tracker in list(trackers.items()): + tracker.setdefault("time_added", time.time()) + tracker.setdefault("time_success", 0) + tracker.setdefault("latency", 99.0) + tracker.setdefault("my", False) + tracker.setdefault("persistent", False) tracker["num_error"] = 0 + if tracker["my"]: + del trackers[address] self.recheckValidTrackers() def save(self): From 37627822deedfd3b13c25345a31e59232cf5fc79 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 6 Jul 2019 01:39:32 +0700 Subject: [PATCH 008/114] TrackerList: make the plugin compatible with TrackerShare --- .../disabled-TrackerList/TrackerListPlugin.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/plugins/disabled-TrackerList/TrackerListPlugin.py b/plugins/disabled-TrackerList/TrackerListPlugin.py index c1ced5e1..b3bc6515 100644 --- a/plugins/disabled-TrackerList/TrackerListPlugin.py +++ b/plugins/disabled-TrackerList/TrackerListPlugin.py @@ -56,9 +56,6 @@ class TrackerList(object): self.last_rescan_failed = False def reload(self): - if "AnnounceShare" not in PluginManager.plugin_manager.plugin_names: - return - rescan_interval = config.tracker_list_rescan_interval if self.last_rescan_failed: rescan_interval = (rescan_interval / 2, 60) @@ -68,14 +65,19 @@ class TrackerList(object): self.last_rescan_time = time.time() - try: - if "tracker_storage" not in locals(): - from AnnounceShare.AnnounceSharePlugin import tracker_storage - self.tracker_storage = tracker_storage - if self.tracker_storage: - gevent.spawn(self.do_rescan) - except Exception as err: - self.log.error("%s" % Debug.formatException(err)) + if "tracker_storage" not in locals(): + try: + if "TrackerShare" in PluginManager.plugin_manager.plugin_names: + from TrackerShare.TrackerSharePlugin import tracker_storage + self.tracker_storage = tracker_storage + elif "AnnounceShare" in PluginManager.plugin_manager.plugin_names: + from AnnounceShare.AnnounceSharePlugin import tracker_storage + self.tracker_storage = tracker_storage + except Exception as err: + self.log.error("%s" % Debug.formatException(err)) + + if self.tracker_storage: + gevent.spawn(self.do_rescan) if "tracker_list" not in locals(): From 6ee1db4197a5a846ba12e461a2a92e9de4c6f6ed Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 6 Jul 2019 17:21:14 +0700 Subject: [PATCH 009/114] TrackerList: fix a typo --- plugins/disabled-TrackerList/TrackerListPlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/disabled-TrackerList/TrackerListPlugin.py b/plugins/disabled-TrackerList/TrackerListPlugin.py index b3bc6515..b3611173 100644 --- a/plugins/disabled-TrackerList/TrackerListPlugin.py +++ b/plugins/disabled-TrackerList/TrackerListPlugin.py @@ -58,7 +58,7 @@ class TrackerList(object): def reload(self): rescan_interval = config.tracker_list_rescan_interval if self.last_rescan_failed: - rescan_interval = (rescan_interval / 2, 60) + rescan_interval = rescan_interval / 2 if self.last_rescan_time > time.time() - rescan_interval: return From aa6d7a468d1b0d225a4a2cef53101c64c99f922f Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sun, 7 Jul 2019 14:31:13 +0700 Subject: [PATCH 010/114] TrackerShare: store trackers in shared-trackers.json, since trackers.json is in use by AnnounceShare and may have an incompatible format in the future --- plugins/TrackerShare/TrackerSharePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index e683dbfc..558163ed 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -25,7 +25,7 @@ class TrackerStorage(object): self.shared_trackers_limit_per_protocol = {} self.shared_trackers_limit_per_protocol["other"] = 2 - self.file_path = "%s/trackers.json" % config.data_dir + self.file_path = "%s/shared-trackers.json" % config.data_dir self.load() self.time_discover = 0.0 self.time_success = 0.0 From b7550474a568cc24a0334679f17115a26956f7dd Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 12 Jul 2019 01:48:52 +0700 Subject: [PATCH 011/114] TrackerZero: copy the Bootstrapper code to a new plugin TrackerZero --- plugins/TrackerZero/TrackerZeroDb.py | 156 +++++++++++++++++++++++ plugins/TrackerZero/TrackerZeroPlugin.py | 155 ++++++++++++++++++++++ plugins/TrackerZero/__init__.py | 1 + 3 files changed, 312 insertions(+) create mode 100644 plugins/TrackerZero/TrackerZeroDb.py create mode 100644 plugins/TrackerZero/TrackerZeroPlugin.py create mode 100644 plugins/TrackerZero/__init__.py diff --git a/plugins/TrackerZero/TrackerZeroDb.py b/plugins/TrackerZero/TrackerZeroDb.py new file mode 100644 index 00000000..b5b57afa --- /dev/null +++ b/plugins/TrackerZero/TrackerZeroDb.py @@ -0,0 +1,156 @@ +import time +import re + +import gevent + +from Config import config +from Db import Db +from util import helper + + +class TrackerZeroDb(Db.Db): + def __init__(self): + self.version = 7 + self.hash_ids = {} # hash -> id cache + super(TrackerZeroDb, self).__init__({"db_name": "TrackerZero"}, "%s/tracker-zero.db" % config.data_dir) + self.foreign_keys = True + self.checkTables() + self.updateHashCache() + gevent.spawn(self.cleanup) + + def cleanup(self): + while 1: + time.sleep(4 * 60) + timeout = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 40)) + self.execute("DELETE FROM peer WHERE date_announced < ?", [timeout]) + + def updateHashCache(self): + res = self.execute("SELECT * FROM hash") + self.hash_ids = {str(row["hash"]): row["hash_id"] for row in res} + self.log.debug("Loaded %s hash_ids" % len(self.hash_ids)) + + def checkTables(self): + version = int(self.execute("PRAGMA user_version").fetchone()[0]) + self.log.debug("Db version: %s, needed: %s" % (version, self.version)) + if version < self.version: + self.createTables() + else: + self.execute("VACUUM") + + def createTables(self): + # Delete all tables + self.execute("PRAGMA writable_schema = 1") + self.execute("DELETE FROM sqlite_master WHERE type IN ('table', 'index', 'trigger')") + self.execute("PRAGMA writable_schema = 0") + self.execute("VACUUM") + self.execute("PRAGMA INTEGRITY_CHECK") + # Create new tables + self.execute(""" + CREATE TABLE peer ( + peer_id INTEGER PRIMARY KEY ASC AUTOINCREMENT NOT NULL UNIQUE, + type TEXT, + address TEXT, + port INTEGER NOT NULL, + date_added DATETIME DEFAULT (CURRENT_TIMESTAMP), + date_announced DATETIME DEFAULT (CURRENT_TIMESTAMP) + ); + """) + self.execute("CREATE UNIQUE INDEX peer_key ON peer (address, port);") + + self.execute(""" + CREATE TABLE peer_to_hash ( + peer_to_hash_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE NOT NULL, + peer_id INTEGER REFERENCES peer (peer_id) ON DELETE CASCADE, + hash_id INTEGER REFERENCES hash (hash_id) + ); + """) + self.execute("CREATE INDEX peer_id ON peer_to_hash (peer_id);") + self.execute("CREATE INDEX hash_id ON peer_to_hash (hash_id);") + + self.execute(""" + CREATE TABLE hash ( + hash_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE NOT NULL, + hash BLOB UNIQUE NOT NULL, + date_added DATETIME DEFAULT (CURRENT_TIMESTAMP) + ); + """) + self.execute("PRAGMA user_version = %s" % self.version) + + def getHashId(self, hash): + if hash not in self.hash_ids: + self.log.debug("New hash: %s" % repr(hash)) + self.execute("INSERT OR IGNORE INTO hash ?", {"hash": hash}) + self.hash_ids[hash] = self.cur.cursor.lastrowid + return self.hash_ids[hash] + + def peerAnnounce(self, ip_type, address, port=None, hashes=[], onion_signed=False, delete_missing_hashes=False): + hashes_ids_announced = [] + for hash in hashes: + hashes_ids_announced.append(self.getHashId(hash)) + + # Check user + res = self.execute("SELECT peer_id FROM peer WHERE ? LIMIT 1", {"address": address, "port": port}) + + user_row = res.fetchone() + now = time.strftime("%Y-%m-%d %H:%M:%S") + if user_row: + peer_id = user_row["peer_id"] + self.execute("UPDATE peer SET date_announced = ? WHERE peer_id = ?", (now, peer_id)) + else: + self.log.debug("New peer: %s signed: %s" % (address, onion_signed)) + if ip_type == "onion" and not onion_signed: + return len(hashes) + self.execute("INSERT INTO peer ?", {"type": ip_type, "address": address, "port": port, "date_announced": now}) + peer_id = self.cur.cursor.lastrowid + + # Check user's hashes + res = self.execute("SELECT * FROM peer_to_hash WHERE ?", {"peer_id": peer_id}) + hash_ids_db = [row["hash_id"] for row in res] + if hash_ids_db != hashes_ids_announced: + hash_ids_added = set(hashes_ids_announced) - set(hash_ids_db) + hash_ids_removed = set(hash_ids_db) - set(hashes_ids_announced) + if ip_type != "onion" or onion_signed: + for hash_id in hash_ids_added: + self.execute("INSERT INTO peer_to_hash ?", {"peer_id": peer_id, "hash_id": hash_id}) + if hash_ids_removed and delete_missing_hashes: + self.execute("DELETE FROM peer_to_hash WHERE ?", {"peer_id": peer_id, "hash_id": list(hash_ids_removed)}) + + return len(hash_ids_added) + len(hash_ids_removed) + else: + return 0 + + def peerList(self, hash, address=None, onions=[], port=None, limit=30, need_types=["ipv4", "onion"], order=True): + back = {"ipv4": [], "ipv6": [], "onion": []} + if limit == 0: + return back + hashid = self.getHashId(hash) + + if order: + order_sql = "ORDER BY date_announced DESC" + else: + order_sql = "" + where_sql = "hash_id = :hashid" + if onions: + onions_escaped = ["'%s'" % re.sub("[^a-z0-9,]", "", onion) for onion in onions if type(onion) is str] + where_sql += " AND address NOT IN (%s)" % ",".join(onions_escaped) + elif address: + where_sql += " AND NOT (address = :address AND port = :port)" + + query = """ + SELECT type, address, port + FROM peer_to_hash + LEFT JOIN peer USING (peer_id) + WHERE %s + %s + LIMIT :limit + """ % (where_sql, order_sql) + res = self.execute(query, {"hashid": hashid, "address": address, "port": port, "limit": limit}) + + for row in res: + if row["type"] in need_types: + if row["type"] == "onion": + packed = helper.packOnionAddress(row["address"], row["port"]) + else: + packed = helper.packAddress(str(row["address"]), row["port"]) + back[row["type"]].append(packed) + return back diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py new file mode 100644 index 00000000..ef455776 --- /dev/null +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -0,0 +1,155 @@ +import time + +from util import helper + +from Plugin import PluginManager +from .TrackerZeroDb import TrackerZeroDb +from Crypt import CryptRsa +from Config import config + +if "db" not in locals().keys(): # Share during reloads + db = TrackerZeroDb() + + +@PluginManager.registerTo("FileRequest") +class FileRequestPlugin(object): + def checkOnionSigns(self, onions, onion_signs, onion_sign_this): + if not onion_signs or len(onion_signs) != len(set(onions)): + return False + + if time.time() - float(onion_sign_this) > 3 * 60: + return False # Signed out of allowed 3 minutes + + onions_signed = [] + # Check onion signs + for onion_publickey, onion_sign in onion_signs.items(): + if CryptRsa.verify(onion_sign_this.encode(), onion_publickey, onion_sign): + onions_signed.append(CryptRsa.publickeyToOnion(onion_publickey)) + else: + break + + # Check if the same onion addresses signed as the announced onces + if sorted(onions_signed) == sorted(set(onions)): + return True + else: + return False + + def actionAnnounce(self, params): + time_started = time.time() + s = time.time() + # Backward compatibility + if "ip4" in params["add"]: + params["add"].append("ipv4") + if "ip4" in params["need_types"]: + params["need_types"].append("ipv4") + + hashes = params["hashes"] + + all_onions_signed = self.checkOnionSigns(params.get("onions", []), params.get("onion_signs"), params.get("onion_sign_this")) + + time_onion_check = time.time() - s + + ip_type = helper.getIpType(self.connection.ip) + + if ip_type == "onion" or self.connection.ip in config.ip_local: + is_port_open = False + elif ip_type in params["add"]: + is_port_open = True + else: + is_port_open = False + + s = time.time() + # Separatley add onions to sites or at once if no onions present + i = 0 + onion_to_hash = {} + for onion in params.get("onions", []): + if onion not in onion_to_hash: + onion_to_hash[onion] = [] + onion_to_hash[onion].append(hashes[i]) + i += 1 + + hashes_changed = 0 + for onion, onion_hashes in onion_to_hash.items(): + hashes_changed += db.peerAnnounce( + ip_type="onion", + address=onion, + port=params["port"], + hashes=onion_hashes, + onion_signed=all_onions_signed + ) + time_db_onion = time.time() - s + + s = time.time() + + if is_port_open: + hashes_changed += db.peerAnnounce( + ip_type=ip_type, + address=self.connection.ip, + port=params["port"], + hashes=hashes, + delete_missing_hashes=params.get("delete") + ) + time_db_ip = time.time() - s + + s = time.time() + # Query sites + back = {} + peers = [] + if params.get("onions") and not all_onions_signed and hashes_changed: + back["onion_sign_this"] = "%.0f" % time.time() # Send back nonce for signing + + if len(hashes) > 500 or not hashes_changed: + limit = 5 + order = False + else: + limit = 30 + order = True + for hash in hashes: + if time.time() - time_started > 1: # 1 sec limit on request + self.connection.log("Announce time limit exceeded after %s/%s sites" % (len(peers), len(hashes))) + break + + hash_peers = db.peerList( + hash, + address=self.connection.ip, onions=list(onion_to_hash.keys()), port=params["port"], + limit=min(limit, params["need_num"]), need_types=params["need_types"], order=order + ) + if "ip4" in params["need_types"]: # Backward compatibility + hash_peers["ip4"] = hash_peers["ipv4"] + del(hash_peers["ipv4"]) + peers.append(hash_peers) + time_peerlist = time.time() - s + + back["peers"] = peers + self.connection.log( + "Announce %s sites (onions: %s, onion_check: %.3fs, db_onion: %.3fs, db_ip: %.3fs, peerlist: %.3fs, limit: %s)" % + (len(hashes), len(onion_to_hash), time_onion_check, time_db_onion, time_db_ip, time_peerlist, limit) + ) + self.response(back) + + +@PluginManager.registerTo("UiRequest") +class UiRequestPlugin(object): + def actionStatsTrackerZero(self): + self.sendHeader() + + # Style + yield """ + + """ + + hash_rows = db.execute("SELECT * FROM hash").fetchall() + for hash_row in hash_rows: + peer_rows = db.execute( + "SELECT * FROM peer LEFT JOIN peer_to_hash USING (peer_id) WHERE hash_id = :hash_id", + {"hash_id": hash_row["hash_id"]} + ).fetchall() + + yield "
%s (added: %s, peers: %s)
" % ( + str(hash_row["hash"]).encode("hex"), hash_row["date_added"], len(peer_rows) + ) + for peer_row in peer_rows: + yield " - {ip4: <30} {onion: <30} added: {date_added}, announced: {date_announced}
".format(**dict(peer_row)) diff --git a/plugins/TrackerZero/__init__.py b/plugins/TrackerZero/__init__.py new file mode 100644 index 00000000..638a8f8a --- /dev/null +++ b/plugins/TrackerZero/__init__.py @@ -0,0 +1 @@ +from . import TrackerZeroPlugin \ No newline at end of file From f4708d9781bfe95522b01e351867fb81ba102177 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 12 Jul 2019 15:44:23 +0700 Subject: [PATCH 012/114] TrackerZero: add a separate class for not to run complicated code in overloaded methods --- plugins/TrackerZero/TrackerZeroPlugin.py | 38 ++++++++++++++++-------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py index ef455776..7b4d3de4 100644 --- a/plugins/TrackerZero/TrackerZeroPlugin.py +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -7,12 +7,10 @@ from .TrackerZeroDb import TrackerZeroDb from Crypt import CryptRsa from Config import config -if "db" not in locals().keys(): # Share during reloads - db = TrackerZeroDb() +class TrackerZero(object): + def __init__(self): + self.log = logging.getLogger("TrackerZero") - -@PluginManager.registerTo("FileRequest") -class FileRequestPlugin(object): def checkOnionSigns(self, onions, onion_signs, onion_sign_this): if not onion_signs or len(onion_signs) != len(set(onions)): return False @@ -34,7 +32,7 @@ class FileRequestPlugin(object): else: return False - def actionAnnounce(self, params): + def actionAnnounce(self, file_request, params): time_started = time.time() s = time.time() # Backward compatibility @@ -49,9 +47,9 @@ class FileRequestPlugin(object): time_onion_check = time.time() - s - ip_type = helper.getIpType(self.connection.ip) + ip_type = helper.getIpType(file_request.connection.ip) - if ip_type == "onion" or self.connection.ip in config.ip_local: + if ip_type == "onion" or file_request.connection.ip in config.ip_local: is_port_open = False elif ip_type in params["add"]: is_port_open = True @@ -84,7 +82,7 @@ class FileRequestPlugin(object): if is_port_open: hashes_changed += db.peerAnnounce( ip_type=ip_type, - address=self.connection.ip, + address=file_request.connection.ip, port=params["port"], hashes=hashes, delete_missing_hashes=params.get("delete") @@ -106,12 +104,12 @@ class FileRequestPlugin(object): order = True for hash in hashes: if time.time() - time_started > 1: # 1 sec limit on request - self.connection.log("Announce time limit exceeded after %s/%s sites" % (len(peers), len(hashes))) + file_request.connection.log("Announce time limit exceeded after %s/%s sites" % (len(peers), len(hashes))) break hash_peers = db.peerList( hash, - address=self.connection.ip, onions=list(onion_to_hash.keys()), port=params["port"], + address=file_request.connection.ip, onions=list(onion_to_hash.keys()), port=params["port"], limit=min(limit, params["need_num"]), need_types=params["need_types"], order=order ) if "ip4" in params["need_types"]: # Backward compatibility @@ -121,11 +119,25 @@ class FileRequestPlugin(object): time_peerlist = time.time() - s back["peers"] = peers - self.connection.log( + file_request.connection.log( "Announce %s sites (onions: %s, onion_check: %.3fs, db_onion: %.3fs, db_ip: %.3fs, peerlist: %.3fs, limit: %s)" % (len(hashes), len(onion_to_hash), time_onion_check, time_db_onion, time_db_ip, time_peerlist, limit) ) - self.response(back) + file_request.response(back) + + +if "db" not in locals().keys(): # Share during reloads + db = TrackerZeroDb() + +if "TrackerZero" not in locals(): + tracker_zero = TrackerZero() + + + +@PluginManager.registerTo("FileRequest") +class FileRequestPlugin(object): + def actionAnnounce(self, params): + tracker_zero.actionAnnounce(self, params) @PluginManager.registerTo("UiRequest") From 9a8519b4872c86326043b684aa70c5e86f540ca8 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 13 Jul 2019 00:51:31 +0700 Subject: [PATCH 013/114] TrackerZero: read settings from tracker-zero.json; register listened addresses in TrackerShare/AnnounceShare --- plugins/TrackerZero/TrackerZeroPlugin.py | 112 ++++++++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py index 7b4d3de4..53ca6d6c 100644 --- a/plugins/TrackerZero/TrackerZeroPlugin.py +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -1,3 +1,7 @@ +import atexit +import json +import logging +import os import time from util import helper @@ -10,6 +14,53 @@ from Config import config class TrackerZero(object): def __init__(self): self.log = logging.getLogger("TrackerZero") + self.log_once = set() + self.enabled_addresses = [] + self.config_file_path = "%s/tracker-zero.json" % config.data_dir + self.config = None + self.load() + atexit.register(self.save) + + + def logOnce(self, message): + if message in self.log_once: + return + self.log_once.add(message) + self.log.info(message) + + def getDefaultConfig(self): + return { + "settings": { + "enable": False, + "enable_only_in_tor_always_mode": True, + "listen_on_public_ips": False, + "listen_on_temporary_onion_address": False, + "listen_on_persistent_onion_address": True + } + } + + def readJSON(self, file_path, default_value): + if not os.path.isfile(file_path): + try: + self.writeJSON(file_path, default_value) + except Exception as err: + self.log.error("Error writing %s: %s" % (file_path, err)) + return default_value + + try: + return json.load(open(file_path)) + except Exception as err: + self.log.error("Error loading %s: %s" % (file_path, err)) + return default_value + + def writeJSON(self, file_path, value): + helper.atomicWrite(file_path, json.dumps(value, indent=2, sort_keys=True).encode("utf8")) + + def load(self): + self.config = self.readJSON(self.config_file_path, self.getDefaultConfig()) + + def save(self): + self.writeJSON(self.config_file_path, self.config) def checkOnionSigns(self, onions, onion_signs, onion_sign_this): if not onion_signs or len(onion_signs) != len(set(onions)): @@ -126,10 +177,61 @@ class TrackerZero(object): file_request.response(back) + def getTrackerStorage(self): + try: + if "TrackerShare" in PluginManager.plugin_manager.plugin_names: + from TrackerShare.TrackerSharePlugin import tracker_storage + return tracker_storage + elif "AnnounceShare" in PluginManager.plugin_manager.plugin_names: + from AnnounceShare.AnnounceSharePlugin import tracker_storage + return tracker_storage + except Exception as err: + self.log.error("%s" % Debug.formatException(err)) + + return None + + def registerSharedAddresses(self, file_server, port_open): + tracker_storage = self.getTrackerStorage() + if not tracker_storage: + return + + settings = self.config.get("settings", {}) + + if not settings.get("enable"): + self.logOnce("Plugin loaded, but disabled by the settings") + return False + + if settings.get("enable_only_in_tor_always_mode") and not config.tor == "always": + self.logOnce("Plugin loaded, but disabled from running in the modes other than 'tor = always'") + return False + + self.enabled_addresses = [] + + if settings.get("listen_on_public_ips") and port_open and not config.tor == "always": + for ip in file_server.ip_external_list: + my_tracker_address = "zero://%s:%s" % (ip, config.fileserver_port) + if tracker_storage.onTrackerFound(my_tracker_address, my=True): + self.logOnce("listening on public IP: %s" % my_tracker_address) + self.enabled_addresses.append(my_tracker_address) + + if settings.get("listen_on_temporary_onion_address") and file_server.tor_manager.enabled: + onion = file_server.tor_manager.getOnion(config.homepage) + if onion: + my_tracker_address = "zero://%s.onion:%s" % (onion, file_server.tor_manager.fileserver_port) + if tracker_storage.onTrackerFound(my_tracker_address, my=True): + self.logOnce("listening on temporary onion address: %s" % my_tracker_address) + self.enabled_addresses.append(my_tracker_address) + + if settings.get("listen_on_persistent_onion_address") and file_server.tor_manager.enabled: + # FIXME: not implemented + pass + + return len(self.enabled_addresses) > 0 + if "db" not in locals().keys(): # Share during reloads db = TrackerZeroDb() -if "TrackerZero" not in locals(): +if "tracker_zero" not in locals(): tracker_zero = TrackerZero() @@ -140,6 +242,14 @@ class FileRequestPlugin(object): tracker_zero.actionAnnounce(self, params) +@PluginManager.registerTo("FileServer") +class FileServerPlugin(object): + def portCheck(self, *args, **kwargs): + res = super(FileServerPlugin, self).portCheck(*args, **kwargs) + tracker_zero.registerSharedAddresses(self, res) + return res + + @PluginManager.registerTo("UiRequest") class UiRequestPlugin(object): def actionStatsTrackerZero(self): From a36b2c924165cd44ddc8715a0bdae660321b2a31 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 13 Jul 2019 02:34:07 +0700 Subject: [PATCH 014/114] TrackerZero: add support of persistent onion addresses --- plugins/TrackerZero/TrackerZeroPlugin.py | 62 +++++++++++++++++------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py index 53ca6d6c..f31bd09b 100644 --- a/plugins/TrackerZero/TrackerZeroPlugin.py +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -1,6 +1,7 @@ import atexit import json import logging +import re import os import time @@ -21,7 +22,6 @@ class TrackerZero(object): self.load() atexit.register(self.save) - def logOnce(self, message): if message in self.log_once: return @@ -190,11 +190,23 @@ class TrackerZero(object): return None - def registerSharedAddresses(self, file_server, port_open): - tracker_storage = self.getTrackerStorage() - if not tracker_storage: + def registerTrackerAddress(self, message, address, port): + _tracker_storage = self.getTrackerStorage() + if not _tracker_storage: return + my_tracker_address = "zero://%s:%s" % (address, port) + if _tracker_storage.onTrackerFound(my_tracker_address, my=True): + self.logOnce("listening on %s: %s" % (message, my_tracker_address)) + self.enabled_addresses.append("%s:%s" % (address, port)) + + def registerTrackerAddresses(self, file_server, port_open): + _tracker_storage = self.getTrackerStorage() + if not _tracker_storage: + return + + tor_manager = file_server.tor_manager + settings = self.config.get("settings", {}) if not settings.get("enable"): @@ -209,22 +221,36 @@ class TrackerZero(object): if settings.get("listen_on_public_ips") and port_open and not config.tor == "always": for ip in file_server.ip_external_list: - my_tracker_address = "zero://%s:%s" % (ip, config.fileserver_port) - if tracker_storage.onTrackerFound(my_tracker_address, my=True): - self.logOnce("listening on public IP: %s" % my_tracker_address) - self.enabled_addresses.append(my_tracker_address) + self.registerTrackerAddress("public IP", ip, config.fileserver_port) - if settings.get("listen_on_temporary_onion_address") and file_server.tor_manager.enabled: - onion = file_server.tor_manager.getOnion(config.homepage) + if settings.get("listen_on_temporary_onion_address") and tor_manager.enabled: + onion = tor_manager.getOnion(config.homepage) if onion: - my_tracker_address = "zero://%s.onion:%s" % (onion, file_server.tor_manager.fileserver_port) - if tracker_storage.onTrackerFound(my_tracker_address, my=True): - self.logOnce("listening on temporary onion address: %s" % my_tracker_address) - self.enabled_addresses.append(my_tracker_address) + self.registerTrackerAddress("temporary onion address", onion, tor_manager.fileserver_port) - if settings.get("listen_on_persistent_onion_address") and file_server.tor_manager.enabled: - # FIXME: not implemented - pass + if settings.get("listen_on_persistent_onion_address") and tor_manager.enabled: + persistent_addresses = self.config.setdefault("persistent_addresses", {}) + if len(persistent_addresses) == 0: + result = tor_manager.makeOnionAndKey() + if result: + onion_address, onion_privatekey = result + persistent_addresses[onion_address] = { + "private_key": onion_privatekey + } + self.registerTrackerAddress("persistent onion address", onion_address, tor_manager.fileserver_port) + else: + for address, d in persistent_addresses.items(): + private_key = d.get("private_key") + if not private_key: + continue + res = tor_manager.request( + "ADD_ONION RSA1024:%s port=%s" % (private_key, tor_manager.fileserver_port) + ) + match = re.search("ServiceID=([A-Za-z0-9]+)", res) + if match: + onion_address = match.groups()[0] + if onion_address == address: + self.registerTrackerAddress("persistent onion address", onion_address, tor_manager.fileserver_port) return len(self.enabled_addresses) > 0 @@ -246,7 +272,7 @@ class FileRequestPlugin(object): class FileServerPlugin(object): def portCheck(self, *args, **kwargs): res = super(FileServerPlugin, self).portCheck(*args, **kwargs) - tracker_zero.registerSharedAddresses(self, res) + tracker_zero.registerTrackerAddresses(self, res) return res From eb6d0c9644980b7e8a85455f543c98675729f589 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 13 Jul 2019 15:35:16 +0700 Subject: [PATCH 015/114] TrackerZero: add missimg @helper.encodeResponse --- plugins/TrackerZero/TrackerZeroPlugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py index f31bd09b..7495eea1 100644 --- a/plugins/TrackerZero/TrackerZeroPlugin.py +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -278,6 +278,8 @@ class FileServerPlugin(object): @PluginManager.registerTo("UiRequest") class UiRequestPlugin(object): + + @helper.encodeResponse def actionStatsTrackerZero(self): self.sendHeader() From d57deaa8e4869b80abca12d027c24bb86a08c054 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 13 Jul 2019 15:36:05 +0700 Subject: [PATCH 016/114] TrackerShare: ignore the udp:// protocol, when UDP is known to be disabled by the config --- plugins/TrackerShare/TrackerSharePlugin.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 558163ed..156f7fd2 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -75,6 +75,9 @@ class TrackerStorage(object): if not self.isTrackerAddressValid(address): del trackers[address] + def isUdpEnabled(self): + return not (config.disable_udp or config.trackers_proxy != "disable") + def getNormalizedTrackerProtocol(self, tracker_address): if not self.site_announcer: return None @@ -111,6 +114,8 @@ class TrackerStorage(object): protocol = self.getNormalizedTrackerProtocol(tracker_address) if not protocol: continue + if protocol == "udp" and not self.isUdpEnabled(): + continue protocols.add(protocol) protocols = list(protocols) From de5a9ff67bfa3c6722987747cca4aead060763f3 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 13 Jul 2019 15:38:05 +0700 Subject: [PATCH 017/114] TrackerShare: drop incomplete support of Bootstrapper, we now have TrackerZero, which is able to register its addresses by itself --- plugins/TrackerShare/TrackerSharePlugin.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 156f7fd2..eb07b978 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -365,17 +365,6 @@ class FileRequestPlugin(object): self.response({"trackers": shared_trackers}) -@PluginManager.registerTo("FileServer") -class FileServerPlugin(object): - def portCheck(self, *args, **kwargs): - res = super(FileServerPlugin, self).portCheck(*args, **kwargs) - if res and not config.tor == "always" and "Bootstrapper" in PluginManager.plugin_manager.plugin_names: - for ip in self.ip_external_list: - my_tracker_address = "zero://%s:%s" % (ip, config.fileserver_port) - tracker_storage.onTrackerFound(my_tracker_address, my=True) - return res - - @PluginManager.registerTo("ConfigPlugin") class ConfigPlugin(object): def createArguments(self): From 142f5862df1ff383c1f409a7efaf3ad466ce4c91 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 13 Jul 2019 15:49:21 +0700 Subject: [PATCH 018/114] TrackerZero: ignore "announce" action if the plugin is disabled by its settings --- plugins/TrackerZero/TrackerZeroPlugin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py index 7495eea1..92070fa5 100644 --- a/plugins/TrackerZero/TrackerZeroPlugin.py +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -84,6 +84,10 @@ class TrackerZero(object): return False def actionAnnounce(self, file_request, params): + if len(self.enabled_addresses) < 1: + file_request.actionUnknown("announce", params) + return + time_started = time.time() s = time.time() # Backward compatibility From fee63a1ed2bf2fc9c4ce24d4a7f3f9824b8765a4 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 13 Jul 2019 21:13:41 +0700 Subject: [PATCH 019/114] TrackerZero: add the missing .onion suffix --- plugins/TrackerZero/TrackerZeroPlugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py index 92070fa5..3ae9531f 100644 --- a/plugins/TrackerZero/TrackerZeroPlugin.py +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -230,7 +230,7 @@ class TrackerZero(object): if settings.get("listen_on_temporary_onion_address") and tor_manager.enabled: onion = tor_manager.getOnion(config.homepage) if onion: - self.registerTrackerAddress("temporary onion address", onion, tor_manager.fileserver_port) + self.registerTrackerAddress("temporary onion address", "%s.onion" % onion, tor_manager.fileserver_port) if settings.get("listen_on_persistent_onion_address") and tor_manager.enabled: persistent_addresses = self.config.setdefault("persistent_addresses", {}) @@ -241,7 +241,7 @@ class TrackerZero(object): persistent_addresses[onion_address] = { "private_key": onion_privatekey } - self.registerTrackerAddress("persistent onion address", onion_address, tor_manager.fileserver_port) + self.registerTrackerAddress("persistent onion address", "%s.onion" % onion_address, tor_manager.fileserver_port) else: for address, d in persistent_addresses.items(): private_key = d.get("private_key") @@ -254,7 +254,7 @@ class TrackerZero(object): if match: onion_address = match.groups()[0] if onion_address == address: - self.registerTrackerAddress("persistent onion address", onion_address, tor_manager.fileserver_port) + self.registerTrackerAddress("persistent onion address", "%s.onion" % onion_address, tor_manager.fileserver_port) return len(self.enabled_addresses) > 0 From 2e1b0e093f938576891cc4e019dbb3b37997056f Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sun, 14 Jul 2019 18:26:38 +0700 Subject: [PATCH 020/114] TrackerZero: fix errors in actionStatsTrackerZero() --- plugins/TrackerZero/TrackerZeroPlugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py index 3ae9531f..52912a61 100644 --- a/plugins/TrackerZero/TrackerZeroPlugin.py +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -4,6 +4,7 @@ import logging import re import os import time +import binascii from util import helper @@ -303,7 +304,7 @@ class UiRequestPlugin(object): ).fetchall() yield "
%s (added: %s, peers: %s)
" % ( - str(hash_row["hash"]).encode("hex"), hash_row["date_added"], len(peer_rows) + binascii.hexlify(hash_row["hash"]).decode("utf-8"), hash_row["date_added"], len(peer_rows) ) for peer_row in peer_rows: - yield " - {ip4: <30} {onion: <30} added: {date_added}, announced: {date_announced}
".format(**dict(peer_row)) + yield " - {type: <6} {address: <30} {port: >5} added: {date_added}, announced: {date_announced}
".format(**dict(peer_row)) From 96e935300ce65c94c215deb89798d8f19ef38ef3 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 15 Jul 2019 14:13:14 +0700 Subject: [PATCH 021/114] TrackerZero: fix error in updateHashCache() Reported to the upstream: https://github.com/HelloZeroNet/ZeroNet/issues/2095 --- plugins/TrackerZero/TrackerZeroDb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/TrackerZero/TrackerZeroDb.py b/plugins/TrackerZero/TrackerZeroDb.py index b5b57afa..e91cf9be 100644 --- a/plugins/TrackerZero/TrackerZeroDb.py +++ b/plugins/TrackerZero/TrackerZeroDb.py @@ -26,7 +26,7 @@ class TrackerZeroDb(Db.Db): def updateHashCache(self): res = self.execute("SELECT * FROM hash") - self.hash_ids = {str(row["hash"]): row["hash_id"] for row in res} + self.hash_ids = {row["hash"]: row["hash_id"] for row in res} self.log.debug("Loaded %s hash_ids" % len(self.hash_ids)) def checkTables(self): From 2811d7c9d4963323c9e12d030d48b46154bd1927 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 17 Jul 2019 23:21:11 +0700 Subject: [PATCH 022/114] TrackerShare: replace self.site_announcer on every call from SiteAnnouncer --- plugins/TrackerShare/TrackerSharePlugin.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index eb07b978..6ae642d1 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -47,11 +47,15 @@ class TrackerStorage(object): return l.get(protocol, l.get("other")) def setSiteAnnouncer(self, site_announcer): - if self.site_announcer: + if not site_announcer: return - self.site_announcer = site_announcer - self.initTrackerLimitForProtocol() - self.recheckValidTrackers() + + if not self.site_announcer: + self.site_announcer = site_announcer + self.initTrackerLimitForProtocol() + self.recheckValidTrackers() + else: + self.site_announcer = site_announcer def isTrackerAddressValid(self, tracker_address): if not self.site_announcer: # Not completely initialized, skip check From 0d02c3c4dabb82ae918f0d5814520a571e848890 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 17 Jul 2019 23:23:03 +0700 Subject: [PATCH 023/114] TrackerShare: increase default limits: zero=10,other=5; total=20 --- plugins/TrackerShare/TrackerSharePlugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 6ae642d1..65cf99b5 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -373,7 +373,7 @@ class FileRequestPlugin(object): class ConfigPlugin(object): def createArguments(self): group = self.parser.add_argument_group("TrackerShare plugin") - group.add_argument('--shared_trackers_limit', help='Discover new shared trackers if this number of shared trackers isn\'t reached (total)', default=10, type=int, metavar='limit') - group.add_argument('--shared_trackers_limit_per_protocol', help='Discover new shared trackers if this number of shared trackers isn\'t reached per each supported protocol', default="zero=5,other=4", metavar='limit') + group.add_argument('--shared_trackers_limit', help='Discover new shared trackers if this number of shared trackers isn\'t reached (total)', default=20, type=int, metavar='limit') + group.add_argument('--shared_trackers_limit_per_protocol', help='Discover new shared trackers if this number of shared trackers isn\'t reached per each supported protocol', default="zero=10,other=5", metavar='limit') return super(ConfigPlugin, self).createArguments() From cb363d2f11174c60b3c8d5b81117025f2315a2fb Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 17 Jul 2019 23:35:39 +0700 Subject: [PATCH 024/114] TrackerShare: move the tracker list cleanup code from getSupportedProtocols() to a separate method --- plugins/TrackerShare/TrackerSharePlugin.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 65cf99b5..1e70b4af 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -96,12 +96,7 @@ class TrackerStorage(object): return protocol - def getSupportedProtocols(self): - if not self.site_announcer: - return None - - supported_trackers = self.site_announcer.getSupportedTrackers() - + def deleteUnusedTrackers(self, supported_trackers): # If a tracker is in our list, but is absent from the results of getSupportedTrackers(), # it seems to be supported by software, but forbidden by the settings or network configuration. # We check and remove thoose trackers here, since onTrackerError() is never emitted for them. @@ -113,6 +108,14 @@ class TrackerStorage(object): self.log.info("Tracker %s looks unused, removing." % tracker_address) del trackers[tracker_address] + def getSupportedProtocols(self): + if not self.site_announcer: + return None + + supported_trackers = self.site_announcer.getSupportedTrackers() + + self.deleteUnusedTrackers(supported_trackers) + protocols = set() for tracker_address in supported_trackers: protocol = self.getNormalizedTrackerProtocol(tracker_address) From 95c8f0e97e015695bd619a13382dbcf58fa2aa04 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 17 Jul 2019 23:48:53 +0700 Subject: [PATCH 025/114] TrackerShare: consider UDP disabled if config.tor == "always" --- plugins/TrackerShare/TrackerSharePlugin.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 1e70b4af..f213353b 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -80,7 +80,16 @@ class TrackerStorage(object): del trackers[address] def isUdpEnabled(self): - return not (config.disable_udp or config.trackers_proxy != "disable") + if config.disable_udp: + return False + + if config.trackers_proxy != "disable": + return False + + if config.tor == "always": + return False + + return True def getNormalizedTrackerProtocol(self, tracker_address): if not self.site_announcer: From 5f6589cfc231ade0b06e8b29238ec09b06f4ae95 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 18 Jul 2019 00:01:56 +0700 Subject: [PATCH 026/114] TrackerShare: be more verbose in enoughWorkingTrackers() --- plugins/TrackerShare/TrackerSharePlugin.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index f213353b..31627595 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -289,6 +289,8 @@ class TrackerStorage(object): if not trackers_per_protocol: return False + unmet_conditions = 0 + total_nr = 0 for protocol in supported_protocols: @@ -296,15 +298,15 @@ class TrackerStorage(object): if len(trackers) < self.getTrackerLimitForProtocol(protocol): self.log.info("Not enough working trackers for protocol %s: %s < %s" % ( protocol, len(trackers), self.getTrackerLimitForProtocol(protocol))) - return False + unmet_conditions += 1 total_nr += len(trackers) if total_nr < config.shared_trackers_limit: - self.log.info("Not enough working trackers: %s < %s" % ( + self.log.info("Not enough working trackers (total): %s < %s" % ( total_nr, config.shared_trackers_limit)) - return False + unmet_conditions + 1 - return True + return unmet_conditions == 0 def discoverTrackers(self, peers): if self.enoughWorkingTrackers(type="shared"): From f9706e3dc4912bcca1952de44dfbc83a57ca9299 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 9 Mar 2020 19:24:57 +0700 Subject: [PATCH 027/114] TrackerZero: don't register the same onion addresses multiple times --- plugins/TrackerZero/TrackerZeroPlugin.py | 30 +++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py index 52912a61..e90f085d 100644 --- a/plugins/TrackerZero/TrackerZeroPlugin.py +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -18,11 +18,30 @@ class TrackerZero(object): self.log = logging.getLogger("TrackerZero") self.log_once = set() self.enabled_addresses = [] + self.added_onions = set() self.config_file_path = "%s/tracker-zero.json" % config.data_dir self.config = None self.load() atexit.register(self.save) + def addOnion(self, tor_manager, onion, private_key): + # XXX: TorManager hangs if Tor returns a code different from 250 OK, + # so we keep the list of already added onions to avoid adding them twice. + # TODO: Report to the upstream. + + if onion in self.added_onions: + return onion + + res = tor_manager.request( + "ADD_ONION RSA1024:%s port=%s" % (private_key, tor_manager.fileserver_port) + ) + match = re.search("ServiceID=([A-Za-z0-9]+)", res) + if match: + onion_address = match.groups()[0] + self.added_onions.add(onion_address) + return onion_address + return None + def logOnce(self, message): if message in self.log_once: return @@ -248,14 +267,9 @@ class TrackerZero(object): private_key = d.get("private_key") if not private_key: continue - res = tor_manager.request( - "ADD_ONION RSA1024:%s port=%s" % (private_key, tor_manager.fileserver_port) - ) - match = re.search("ServiceID=([A-Za-z0-9]+)", res) - if match: - onion_address = match.groups()[0] - if onion_address == address: - self.registerTrackerAddress("persistent onion address", "%s.onion" % onion_address, tor_manager.fileserver_port) + onion_address = self.addOnion(tor_manager, address, private_key) + if onion_address == address: + self.registerTrackerAddress("persistent onion address", "%s.onion" % onion_address, tor_manager.fileserver_port) return len(self.enabled_addresses) > 0 From 1f34f477efb6585d5563ee2b58a18fb291b0a4ea Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 26 Oct 2020 13:33:25 +0700 Subject: [PATCH 028/114] TrackerShare: add plugin_info.json --- plugins/TrackerShare/plugin_info.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 plugins/TrackerShare/plugin_info.json diff --git a/plugins/TrackerShare/plugin_info.json b/plugins/TrackerShare/plugin_info.json new file mode 100644 index 00000000..9c93cafc --- /dev/null +++ b/plugins/TrackerShare/plugin_info.json @@ -0,0 +1,5 @@ +{ + "name": "TrackerShare", + "description": "Share possible trackers between clients.", + "default": "enabled" +} From b0005026b44870d4efad8664760dfd082e86ab84 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 26 Oct 2020 19:52:35 +0700 Subject: [PATCH 029/114] TrackerShare: rename the `shared` key to `trackers` key in the json file and delete the unused `type` parameter --- plugins/TrackerShare/TrackerSharePlugin.py | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 31627595..a139e82c 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -141,13 +141,13 @@ class TrackerStorage(object): return protocols def getDefaultFile(self): - return {"shared": {}} + return {"trackers": {}} - def onTrackerFound(self, tracker_address, type="shared", my=False, persistent=False): + def onTrackerFound(self, tracker_address, my=False, persistent=False): if not self.isTrackerAddressValid(tracker_address): return False - trackers = self.getTrackers(type) + trackers = self.getTrackers() added = False if tracker_address not in trackers: # "My" trackers never get deleted on announce errors, but aren't saved between restarts. @@ -212,8 +212,8 @@ class TrackerStorage(object): self.log.info("Tracker %s looks down, removing." % tracker_address) del trackers[tracker_address] - def isTrackerWorking(self, tracker_address, type="shared"): - trackers = self.getTrackers(type) + def isTrackerWorking(self, tracker_address): + trackers = self.getTrackers() tracker = trackers[tracker_address] if not tracker: return False @@ -223,29 +223,29 @@ class TrackerStorage(object): return False - def getTrackers(self, type="shared"): - return self.file_content.setdefault(type, {}) + def getTrackers(self): + return self.file_content.setdefault("trackers", {}) - def getTrackersPerProtocol(self, type="shared", working_only=False): + def getTrackersPerProtocol(self, working_only=False): if not self.site_announcer: return None - trackers = self.getTrackers(type) + trackers = self.getTrackers() trackers_per_protocol = {} for tracker_address in trackers: protocol = self.getNormalizedTrackerProtocol(tracker_address) if not protocol: continue - if not working_only or self.isTrackerWorking(tracker_address, type=type): + if not working_only or self.isTrackerWorking(tracker_address): trackers_per_protocol.setdefault(protocol, []).append(tracker_address) return trackers_per_protocol - def getWorkingTrackers(self, type="shared"): + def getWorkingTrackers(self): trackers = { - key: tracker for key, tracker in self.getTrackers(type).items() - if self.isTrackerWorking(key, type) + key: tracker for key, tracker in self.getTrackers().items() + if self.isTrackerWorking(key) } return trackers @@ -280,12 +280,12 @@ class TrackerStorage(object): helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True).encode("utf8")) self.log.debug("Saved in %.3fs" % (time.time() - s)) - def enoughWorkingTrackers(self, type="shared"): + def enoughWorkingTrackers(self): supported_protocols = self.getSupportedProtocols() if not supported_protocols: return False - trackers_per_protocol = self.getTrackersPerProtocol(type="shared", working_only=True) + trackers_per_protocol = self.getTrackersPerProtocol(working_only=True) if not trackers_per_protocol: return False @@ -309,7 +309,7 @@ class TrackerStorage(object): return unmet_conditions == 0 def discoverTrackers(self, peers): - if self.enoughWorkingTrackers(type="shared"): + if self.enoughWorkingTrackers(): return False self.log.info("Discovering trackers from %s peers..." % len(peers)) @@ -357,7 +357,7 @@ class SiteAnnouncerPlugin(object): tracker_storage.time_discover = time.time() gevent.spawn(tracker_storage.discoverTrackers, self.site.getConnectedPeers()) trackers = super(SiteAnnouncerPlugin, self).getTrackers() - shared_trackers = list(tracker_storage.getTrackers("shared").keys()) + shared_trackers = list(tracker_storage.getTrackers().keys()) if shared_trackers: return trackers + shared_trackers else: @@ -378,7 +378,7 @@ class SiteAnnouncerPlugin(object): @PluginManager.registerTo("FileRequest") class FileRequestPlugin(object): def actionGetTrackers(self, params): - shared_trackers = list(tracker_storage.getWorkingTrackers("shared").keys()) + shared_trackers = list(tracker_storage.getWorkingTrackers().keys()) random.shuffle(shared_trackers) self.response({"trackers": shared_trackers}) From d4239d16f9e9565daa3e1a73ddc7bbb049fa55bf Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 27 Oct 2020 12:08:54 +0700 Subject: [PATCH 030/114] TrackerShare: send `my` trackers in response to actionGetTrackers even if they don't seem working; add `private` field for hiding trackers --- plugins/TrackerShare/TrackerSharePlugin.py | 60 +++++++++++++++++----- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index a139e82c..1619fcd6 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -154,13 +154,15 @@ class TrackerStorage(object): # They are to be used as automatically added addresses from the Bootstrap plugin. # Persistent trackers never get deleted. # They are to be used for entries manually added by the user. + # Private trackers never listed to other peer in response of the getTrackers command trackers[tracker_address] = { "time_added": time.time(), "time_success": 0, "latency": 99.0, "num_error": 0, "my": False, - "persistent": False + "persistent": False, + "private": False } self.log.info("New tracker found: %s" % tracker_address) added = True @@ -212,9 +214,29 @@ class TrackerStorage(object): self.log.info("Tracker %s looks down, removing." % tracker_address) del trackers[tracker_address] - def isTrackerWorking(self, tracker_address): - trackers = self.getTrackers() - tracker = trackers[tracker_address] + # Returns the dict of known trackers. + # If condition is None the returned dict can be modified in place, and the + # modifications is reflected in the underlying storage. + # If condition is a function, the dict if filtered by the function, + # and the returned dict has no connection to the underlying storage. + def getTrackers(self, condition = None): + trackers = self.file_content.setdefault("trackers", {}) + + if condition: + trackers = { + key: tracker for key, tracker in trackers.items() + if condition(key) + } + + return trackers + + def resolveTracker(self, tracker): + if isinstance(tracker, str): + tracker = self.getTrackers().get(tracker, None) + return tracker + + def isTrackerWorking(self, tracker): + tracker = self.resolveTracker(tracker) if not tracker: return False @@ -223,8 +245,24 @@ class TrackerStorage(object): return False - def getTrackers(self): - return self.file_content.setdefault("trackers", {}) + def isTrackerShared(self, tracker): + tracker = self.resolveTracker(tracker) + if not tracker: + return False + + if tracker["private"]: + return False + + if tracker["my"]: + return True + + return self.isTrackerWorking(tracker) + + def getWorkingTrackers(self): + return self.getTrackers(self.isTrackerWorking) + + def getSharedTrackers(self): + return self.getTrackers(self.isTrackerShared) def getTrackersPerProtocol(self, working_only=False): if not self.site_announcer: @@ -242,13 +280,6 @@ class TrackerStorage(object): return trackers_per_protocol - def getWorkingTrackers(self): - trackers = { - key: tracker for key, tracker in self.getTrackers().items() - if self.isTrackerWorking(key) - } - return trackers - def getFileContent(self): if not os.path.isfile(self.file_path): open(self.file_path, "w").write("{}") @@ -270,6 +301,7 @@ class TrackerStorage(object): tracker.setdefault("latency", 99.0) tracker.setdefault("my", False) tracker.setdefault("persistent", False) + tracker.setdefault("private", False) tracker["num_error"] = 0 if tracker["my"]: del trackers[address] @@ -378,7 +410,7 @@ class SiteAnnouncerPlugin(object): @PluginManager.registerTo("FileRequest") class FileRequestPlugin(object): def actionGetTrackers(self, params): - shared_trackers = list(tracker_storage.getWorkingTrackers().keys()) + shared_trackers = list(tracker_storage.getSharedTrackers().keys()) random.shuffle(shared_trackers) self.response({"trackers": shared_trackers}) From 3d68a25e130f070e9a140b3c3477d8cc2408b1ae Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 27 Oct 2020 17:01:40 +0700 Subject: [PATCH 031/114] TrackerShare: refactor --- plugins/TrackerShare/TrackerSharePlugin.py | 109 +++++++++++++++++---- 1 file changed, 88 insertions(+), 21 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 1619fcd6..e6b4454b 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -114,7 +114,7 @@ class TrackerStorage(object): t = max(trackers[tracker_address]["time_added"], trackers[tracker_address]["time_success"]) if tracker_address not in supported_trackers and t < time.time() - self.tracker_down_time_interval: - self.log.info("Tracker %s looks unused, removing." % tracker_address) + self.log.info("Tracker %s seems to be disabled by the configuration, removing." % tracker_address) del trackers[tracker_address] def getSupportedProtocols(self): @@ -158,6 +158,7 @@ class TrackerStorage(object): trackers[tracker_address] = { "time_added": time.time(), "time_success": 0, + "time_error": 0, "latency": 99.0, "num_error": 0, "my": False, @@ -173,32 +174,65 @@ class TrackerStorage(object): return added def onTrackerSuccess(self, tracker_address, latency): - trackers = self.getTrackers() - if tracker_address not in trackers: - return False + tracker = self.resolveTracker(tracker_address) + if not tracker: + return - trackers[tracker_address]["latency"] = latency - trackers[tracker_address]["time_success"] = time.time() - trackers[tracker_address]["num_error"] = 0 + tracker["latency"] = latency + tracker["time_success"] = time.time() + tracker["num_error"] = 0 self.time_success = time.time() def onTrackerError(self, tracker_address): - trackers = self.getTrackers() - if tracker_address not in trackers: - return False - - trackers[tracker_address]["time_error"] = time.time() - trackers[tracker_address]["num_error"] += 1 - - if trackers[tracker_address]["my"] or trackers[tracker_address]["persistent"]: + tracker = self.resolveTracker(tracker_address) + if not tracker: return - if self.time_success < time.time() - self.tracker_down_time_interval / 2: - # Don't drop trackers from the list, if there haven't been any successful announces recently. - # There may be network connectivity issues. + tracker["time_error"] = time.time() + tracker["num_error"] += 1 + + self.considerTrackerDeletion(tracker_address) + + def considerTrackerDeletion(self, tracker_address): + tracker = self.resolveTracker(tracker_address) + if not tracker: return + if tracker["my"] or tracker["persistent"]: + return + + error_limit = self.getSuccessiveErrorLimit(tracker_address) + + if tracker["num_error"] > error_limit: + if self.isTrackerDown(tracker_address): + self.log.info("Tracker %s looks down, removing." % tracker_address) + self.deleteTracker(tracker_address) + elif self.areWayTooManyTrackers(tracker_address): + self.log.info( + "Tracker %s has %d successive errors. Looks like we have too many trackers, so removing." % ( + tracker_address, + tracker["num_error"])) + self.deleteTracker(tracker_address) + + def areWayTooManyTrackers(self, tracker_address): + # Prevent the tracker list overgrowth by hard limiting the maximum size + + protocol = self.getNormalizedTrackerProtocol(tracker_address) or "" + + nr_trackers_for_protocol = len(self.getTrackersPerProtocol().get(protocol, [])) + nr_trackers = len(self.getTrackers()) + + hard_limit_mult = 5 + hard_limit_for_protocol = self.getTrackerLimitForProtocol(protocol) * hard_limit_mult + hard_limit = config.shared_trackers_limit * hard_limit_mult + + if (nr_trackers_for_protocol > hard_limit_for_protocol) and (nr_trackers > hard_limit): + return True + + return False + + def getSuccessiveErrorLimit(self, tracker_address): protocol = self.getNormalizedTrackerProtocol(tracker_address) or "" nr_working_trackers_for_protocol = len(self.getTrackersPerProtocol(working_only=True).get(protocol, [])) @@ -210,9 +244,7 @@ class TrackerStorage(object): if nr_working_trackers >= config.shared_trackers_limit: error_limit = 5 - if trackers[tracker_address]["num_error"] > error_limit and trackers[tracker_address]["time_success"] < time.time() - self.tracker_down_time_interval: - self.log.info("Tracker %s looks down, removing." % tracker_address) - del trackers[tracker_address] + return error_limit # Returns the dict of known trackers. # If condition is None the returned dict can be modified in place, and the @@ -230,11 +262,45 @@ class TrackerStorage(object): return trackers + def deleteTracker(self, tracker): + trackers = self.getTrackers() + if isinstance(tracker, str): + if trackers[tracker]: + del trackers[tracker] + else: + trackers.remove(tracker) + def resolveTracker(self, tracker): if isinstance(tracker, str): tracker = self.getTrackers().get(tracker, None) return tracker + def isTrackerDown(self, tracker): + tracker = self.resolveTracker(tracker) + if not tracker: + return False + + # Don't consider any trackers down if there haven't been any successful announces at all + if self.time_success < 1: + return False + + time_success = max(tracker["time_added"], tracker["time_success"]) + time_error = max(tracker["time_added"], tracker["time_error"]) + + if time_success >= time_error: + return False + + # Deadline is calculated based on the time of the last successful announce, + # not based on the current time. + # There may be network connectivity issues, if there haven't been any + # successful announces recently. + + deadline = self.time_success - self.tracker_down_time_interval + if time_success >= deadline: + return False + + return True + def isTrackerWorking(self, tracker): tracker = self.resolveTracker(tracker) if not tracker: @@ -298,6 +364,7 @@ class TrackerStorage(object): for address, tracker in list(trackers.items()): tracker.setdefault("time_added", time.time()) tracker.setdefault("time_success", 0) + tracker.setdefault("time_error", 0) tracker.setdefault("latency", 99.0) tracker.setdefault("my", False) tracker.setdefault("persistent", False) From b6ae96db5a5bfab3d54d1598b9e8242bf4197d38 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 28 Oct 2020 14:48:30 +0700 Subject: [PATCH 032/114] Implement overriding log levels for separate modules --- src/loglevel_overrides.py | 9 ++++++++ src/main.py | 1 + src/util/SelectiveLogger.py | 43 +++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 src/loglevel_overrides.py create mode 100644 src/util/SelectiveLogger.py diff --git a/src/loglevel_overrides.py b/src/loglevel_overrides.py new file mode 100644 index 00000000..5622e523 --- /dev/null +++ b/src/loglevel_overrides.py @@ -0,0 +1,9 @@ +# This file is for adding rules for selectively enabling debug logging +# when working on the code. +# Add your rules here and skip this file when committing changes. + +#import re +#from util import SelectiveLogger +# +#SelectiveLogger.addLogLevelRaisingRule("ConnServer") +#SelectiveLogger.addLogLevelRaisingRule(re.compile(r'^Site:')) diff --git a/src/main.py b/src/main.py index 7a0188e7..209bb9d2 100644 --- a/src/main.py +++ b/src/main.py @@ -4,6 +4,7 @@ import sys import stat import time import logging +import loglevel_overrides startup_errors = [] def startupError(msg): diff --git a/src/util/SelectiveLogger.py b/src/util/SelectiveLogger.py new file mode 100644 index 00000000..fcdcba0a --- /dev/null +++ b/src/util/SelectiveLogger.py @@ -0,0 +1,43 @@ +import logging +import re + +log_level_raising_rules = [] + +def addLogLevelRaisingRule(rule, level=None): + if level is None: + level = logging.INFO + log_level_raising_rules.append({ + "rule": rule, + "level": level + }) + +def matchLogLevelRaisingRule(name): + for rule in log_level_raising_rules: + if isinstance(rule["rule"], re.Pattern): + if rule["rule"].search(name): + return rule["level"] + else: + if rule["rule"] == name: + return rule["level"] + return None + +class SelectiveLogger(logging.getLoggerClass()): + def __init__(self, name, level=logging.NOTSET): + return super().__init__(name, level) + + def raiseLevel(self, level): + raised_level = matchLogLevelRaisingRule(self.name) + if raised_level is not None: + if level < raised_level: + level = raised_level + return level + + def isEnabledFor(self, level): + level = self.raiseLevel(level) + return super().isEnabledFor(level) + + def _log(self, level, msg, args, **kwargs): + level = self.raiseLevel(level) + return super()._log(level, msg, args, **kwargs) + +logging.setLoggerClass(SelectiveLogger) From f1d91989d54ad70b1cc9d3b80118ab573dcdb716 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 28 Oct 2020 14:51:17 +0700 Subject: [PATCH 033/114] SiteAnnouncer: make use of a separate logger instance, not the Site's logger --- src/Site/SiteAnnouncer.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index 2fd63e82..b50a01fe 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -1,6 +1,7 @@ import random import time import hashlib +import logging import re import collections @@ -24,6 +25,8 @@ global_stats = collections.defaultdict(lambda: collections.defaultdict(int)) class SiteAnnouncer(object): def __init__(self, site): self.site = site + self.log = logging.getLogger("Site:%s SiteAnnouncer" % self.site.address_short) + self.stats = {} self.fileserver_port = config.fileserver_port self.peer_id = self.site.connection_server.peer_id @@ -74,7 +77,7 @@ class SiteAnnouncer(object): if time.time() - self.time_last_announce < 30 and not force: return # No reannouncing within 30 secs if force: - self.site.log.debug("Force reannounce in mode %s" % mode) + self.log.debug("Force reannounce in mode %s" % mode) self.fileserver_port = config.fileserver_port self.time_last_announce = time.time() @@ -82,7 +85,7 @@ class SiteAnnouncer(object): trackers = self.getAnnouncingTrackers(mode) if config.verbose: - self.site.log.debug("Tracker announcing, trackers: %s" % trackers) + self.log.debug("Tracker announcing, trackers: %s" % trackers) errors = [] slow = [] @@ -96,7 +99,7 @@ class SiteAnnouncer(object): time_announce_allowed = time.time() - 60 * min(30, tracker_stats["num_error"]) if tracker_stats["num_error"] > 5 and tracker_stats["time_request"] > time_announce_allowed and not force: if config.verbose: - self.site.log.debug("Tracker %s looks unreliable, announce skipped (error: %s)" % (tracker, tracker_stats["num_error"])) + self.log.debug("Tracker %s looks unreliable, announce skipped (error: %s)" % (tracker, tracker_stats["num_error"])) continue thread = self.site.greenlet_manager.spawn(self.announceTracker, tracker, mode=mode) threads.append(thread) @@ -129,15 +132,15 @@ class SiteAnnouncer(object): else: announced_to = "%s/%s trackers" % (num_announced, len(threads)) if mode != "update" or config.verbose: - self.site.log.debug( + self.log.debug( "Announced in mode %s to %s in %.3fs, errors: %s, slow: %s" % (mode, announced_to, time.time() - s, errors, slow) ) else: if len(threads) > 1: - self.site.log.error("Announce to %s trackers in %.3fs, failed" % (len(threads), time.time() - s)) + self.log.error("Announce to %s trackers in %.3fs, failed" % (len(threads), time.time() - s)) if len(threads) == 1 and mode != "start": # Move to next tracker - self.site.log.debug("Tracker failed, skipping to next one...") + self.log.debug("Tracker failed, skipping to next one...") self.site.greenlet_manager.spawnLater(1.0, self.announce, force=force, mode=mode, pex=pex) self.updateWebsocket(trackers="announced") @@ -177,7 +180,7 @@ class SiteAnnouncer(object): s = time.time() address_parts = self.getAddressParts(tracker) if not address_parts: - self.site.log.warning("Tracker %s error: Invalid address" % tracker) + self.log.warning("Tracker %s error: Invalid address" % tracker) return False if tracker not in self.stats: @@ -188,7 +191,7 @@ class SiteAnnouncer(object): self.stats[tracker]["time_request"] = time.time() global_stats[tracker]["time_request"] = time.time() if config.verbose: - self.site.log.debug("Tracker announcing to %s (mode: %s)" % (tracker, mode)) + self.log.debug("Tracker announcing to %s (mode: %s)" % (tracker, mode)) if mode == "update": num_want = 10 else: @@ -202,7 +205,7 @@ class SiteAnnouncer(object): else: raise AnnounceError("Unknown protocol: %s" % address_parts["protocol"]) except Exception as err: - self.site.log.warning("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode)) + self.log.warning("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode)) error = err if error: @@ -249,7 +252,7 @@ class SiteAnnouncer(object): self.site.updateWebsocket(peers_added=added) if config.verbose: - self.site.log.debug( + self.log.debug( "Tracker result: %s://%s (found %s peers, new: %s, total: %s)" % (address_parts["protocol"], address_parts["address"], len(peers), added, len(self.site.peers)) ) @@ -281,7 +284,7 @@ class SiteAnnouncer(object): time.sleep(0.1) if done == query_num: break - self.site.log.debug("Pex result: from %s peers got %s new peers." % (done, total_added)) + self.log.debug("Pex result: from %s peers got %s new peers." % (done, total_added)) def updateWebsocket(self, **kwargs): if kwargs: From 112c778c28cd9390a9ef714d861a080f99d7fa85 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 28 Oct 2020 14:53:56 +0700 Subject: [PATCH 034/114] Peer.py: allow overriding the log level Not a best solution, but with minimal code changes. --- src/Peer/Peer.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 03cc1f47..d8e28121 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -32,6 +32,8 @@ class Peer(object): self.site = site self.key = "%s:%s" % (ip, port) + self.log_level = logging.DEBUG + self.connection = None self.connection_server = connection_server self.has_hashfield = False # Lazy hashfield object not created yet @@ -59,12 +61,18 @@ class Peer(object): return getattr(self, key) def log(self, text): - if not config.verbose: - return # Only log if we are in debug mode + if self.log_level <= logging.DEBUG: + if not config.verbose: + return # Only log if we are in debug mode + + logger = None + if self.site: - self.site.log.debug("%s:%s %s" % (self.ip, self.port, text)) + logger = self.site.log else: - logging.debug("%s:%s %s" % (self.ip, self.port, text)) + logger = logging.getLogger() + + logger.log(self.log_level, "%s:%s %s" % (self.ip, self.port, text)) # Connect to host def connect(self, connection=None): From 3ca323f8b07dbde15106152922e2770be6f47a24 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 28 Oct 2020 17:56:11 +0700 Subject: [PATCH 035/114] FileServer: move loadTrackersFile() to a separate thread --- src/File/FileServer.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 5f18c645..186d88a3 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -318,11 +318,21 @@ class FileServer(ConnectionServer): site.sendMyHashfield(3) site.updateHashfield(3) + # Periodic reloading of tracker files + def reloadTrackerFilesThread(self): + # TODO: + # This should probably be more sophisticated. + # We should check if the files have actually changed, + # and do it more often. + interval = 60 * 10 + while 1: + time.sleep(interval) + config.loadTrackersFile() + # Announce sites every 20 min def announceSites(self): time.sleep(5 * 60) # Sites already announced on startup while 1: - config.loadTrackersFile() s = time.time() for address, site in list(self.sites.items()): if not site.isServing(): @@ -387,6 +397,7 @@ class FileServer(ConnectionServer): if check_sites: # Open port, Update sites, Check files integrity gevent.spawn(self.checkSites) + thread_reaload_tracker_files = gevent.spawn(self.reloadTrackerFilesThread) thread_announce_sites = gevent.spawn(self.announceSites) thread_cleanup_sites = gevent.spawn(self.cleanupSites) thread_wakeup_watcher = gevent.spawn(self.wakeupWatcher) From 511a90a5c58b617298da557bb2c2bd773175b28f Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 28 Oct 2020 20:31:48 +0700 Subject: [PATCH 036/114] Redesign Site.needConnections() --- src/File/FileServer.py | 10 ++--- src/Peer/Peer.py | 9 +++-- src/Site/Site.py | 87 +++++++++++++++++++++++++++++++++--------- 3 files changed, 80 insertions(+), 26 deletions(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 186d88a3..c1134800 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -295,12 +295,12 @@ class FileServer(ConnectionServer): elif site.bad_files: site.retryBadFiles() - if time.time() - site.settings.get("modified", 0) < 60 * 60 * 24 * 7: - # Keep active connections if site has been modified witin 7 days - connected_num = site.needConnections(check_site_on_reconnect=True) + # Keep active connections + connected_num = site.needConnections(check_site_on_reconnect=True) - if connected_num < config.connected_limit: # This site has small amount of peers, protect them from closing - peers_protected.update([peer.key for peer in site.getConnectedPeers()]) + if connected_num < config.connected_limit: + # This site has small amount of peers, protect them from closing + peers_protected.update([peer.key for peer in site.getConnectedPeers()]) time.sleep(1) # Prevent too quick request diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index d8e28121..75eb190f 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -74,6 +74,9 @@ class Peer(object): logger.log(self.log_level, "%s:%s %s" % (self.ip, self.port, text)) + def isConnected(self): + return self.connection and self.connection.connected + # Connect to host def connect(self, connection=None): if self.reputation < -10: @@ -249,11 +252,11 @@ class Peer(object): return buff # Send a ping request - def ping(self): + def ping(self, timeout=10.0, tryes=3): response_time = None - for retry in range(1, 3): # Retry 3 times + for retry in range(1, tryes): # Retry 3 times s = time.time() - with gevent.Timeout(10.0, False): # 10 sec timeout, don't raise exception + with gevent.Timeout(timeout, False): res = self.request("ping") if res and "body" in res and res["body"] == b"Pong!": diff --git a/src/Site/Site.py b/src/Site/Site.py index 354fe9c0..fff592cd 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -853,36 +853,87 @@ class Site(object): if self.isServing(): self.announcer.announce(*args, **kwargs) - # Keep connections to get the updates - def needConnections(self, num=None, check_site_on_reconnect=False): - if num is None: - if len(self.peers) < 50: - num = 3 - else: - num = 6 - need = min(len(self.peers), num, config.connected_limit) # Need 5 peer, but max total peers + def getPreferableActiveConnectionCount(self): + if not self.isServing(): + return 0 + age = time.time() - self.settings.get("modified", 0) + count = 0 + + if age < 60 * 60: + count = 10 + elif age < 60 * 60 * 5: + count = 8 + elif age < 60 * 60 * 24: + count = 6 + elif age < 60 * 60 * 24 * 3: + count = 4 + elif age < 60 * 60 * 24 * 7: + count = 2 + + if len(self.peers) < 50: + count = max(count, 5) + + return count + + def tryConnectingToMorePeers(self, more=1, pex=True, try_harder=False): + max_peers = more * 2 + 10 + if try_harder: + max_peers += 10000 + + connected = 0 + for peer in self.getRecentPeers(max_peers): + if not peer.isConnected(): + if pex: + peer.pex() + else: + peer.ping(timeout=2.0, tryes=1) + + if peer.isConnected(): + connected += 1 + + if connected >= more: + break + + return connected + + def bringConnections(self, need=1, check_site_on_reconnect=False, pex=True, try_harder=False): connected = len(self.getConnectedPeers()) - connected_before = connected self.log.debug("Need connections: %s, Current: %s, Total: %s" % (need, connected, len(self.peers))) - if connected < need: # Need more than we have - for peer in self.getRecentPeers(30): - if not peer.connection or not peer.connection.connected: # No peer connection or disconnected - peer.pex() # Initiate peer exchange - if peer.connection and peer.connection.connected: - connected += 1 # Successfully connected - if connected >= need: - break + if connected < need: + connected += self.tryConnectingToMorePeers(more=(need-connected), pex=pex, try_harder=try_harder) self.log.debug( "Connected before: %s, after: %s. Check site: %s." % (connected_before, connected, check_site_on_reconnect) ) if check_site_on_reconnect and connected_before == 0 and connected > 0 and self.connection_server.has_internet: - gevent.spawn(self.update, check_files=False) + self.greenlet_manager.spawn(self.update, check_files=False) + + return connected + + # Keep connections + def needConnections(self, num=None, check_site_on_reconnect=False, pex=True): + if num is None: + num = self.getPreferableActiveConnectionCount() + + need = min(len(self.peers), num, config.connected_limit) + + connected = self.bringConnections( + need=need, + check_site_on_reconnect=check_site_on_reconnect, + pex=pex, + try_harder=False) + + if connected < need: + self.greenlet_manager.spawnLater(1.0, self.bringConnections, + need=need, + check_site_on_reconnect=check_site_on_reconnect, + pex=pex, + try_harder=True) return connected From 8fd88c50f923fe2338fc793e7ea5a18473c56e38 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 28 Oct 2020 23:38:17 +0700 Subject: [PATCH 037/114] Redesign cleanupSites() and all the related stuff and rename it to periodic maintenance. --- src/Config.py | 2 +- src/File/FileServer.py | 68 +++++++------------ src/Peer/Peer.py | 26 +++++++- src/Site/Site.py | 125 +++++++++++++++++++++++++---------- src/util/CircularIterator.py | 34 ++++++++++ src/util/__init__.py | 1 + 6 files changed, 175 insertions(+), 81 deletions(-) create mode 100644 src/util/CircularIterator.py diff --git a/src/Config.py b/src/Config.py index 41c914f2..70b284bf 100644 --- a/src/Config.py +++ b/src/Config.py @@ -253,7 +253,7 @@ class Config(object): self.parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, type=int, metavar='limit') self.parser.add_argument('--file_size_limit', help='Maximum per file size limit in MB', default=10, type=int, metavar='limit') - self.parser.add_argument('--connected_limit', help='Max connected peer per site', default=8, type=int, metavar='connected_limit') + self.parser.add_argument('--connected_limit', help='Max connected peer per site', default=10, type=int, metavar='connected_limit') self.parser.add_argument('--global_connected_limit', help='Max connections', default=512, type=int, metavar='global_connected_limit') self.parser.add_argument('--workers', help='Download workers per site', default=5, type=int, metavar='workers') diff --git a/src/File/FileServer.py b/src/File/FileServer.py index c1134800..fbc3cc85 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -10,6 +10,7 @@ from gevent.server import StreamServer import util from util import helper +from util import CircularIterator from Config import config from .FileRequest import FileRequest from Peer import PeerPortchecker @@ -18,7 +19,6 @@ from Connection import ConnectionServer from Plugin import PluginManager from Debug import Debug - @PluginManager.acceptPlugins class FileServer(ConnectionServer): @@ -259,55 +259,35 @@ class FileServer(ConnectionServer): check_thread.join(timeout=5) self.log.debug("Checksites done in %.3fs" % (time.time() - s)) - def cleanupSites(self): + def sitesMaintenanceThread(self): import gc startup = True - time.sleep(5 * 60) # Sites already cleaned up on startup - peers_protected = set([]) + + short_timeout = 2 + long_timeout = 60 * 5 + + circular_iterator = CircularIterator() + while 1: - # Sites health care every 20 min + if circular_iterator.isWrapped(): + time.sleep(long_timeout) + circular_iterator.resetSuccessiveCount() + gc.collect() # Explicit garbage collection + self.log.debug( - "Running site cleanup, connections: %s, internet: %s, protected peers: %s" % - (len(self.connections), self.has_internet, len(peers_protected)) + "Running site cleanup, connections: %s, internet: %s" % + (len(self.connections), self.has_internet) ) - for address, site in list(self.sites.items()): - if not site.isServing(): - continue + site = circular_iterator.next(list(self.sites.values())) + if site: + done = site.runPeriodicMaintenance(startup=startup) + if done: + time.sleep(short_timeout) + site = None - if not startup: - site.cleanupPeers(peers_protected) - - time.sleep(1) # Prevent too quick request - - peers_protected = set([]) - for address, site in list(self.sites.items()): - if not site.isServing(): - continue - - if site.peers: - with gevent.Timeout(10, exception=False): - site.announcer.announcePex() - - # Last check modification failed - if site.content_updated is False: - site.update() - elif site.bad_files: - site.retryBadFiles() - - # Keep active connections - connected_num = site.needConnections(check_site_on_reconnect=True) - - if connected_num < config.connected_limit: - # This site has small amount of peers, protect them from closing - peers_protected.update([peer.key for peer in site.getConnectedPeers()]) - - time.sleep(1) # Prevent too quick request - - site = None - gc.collect() # Implicit garbage collection - startup = False - time.sleep(60 * 20) + if circular_iterator.isWrapped(): + startup = False def announceSite(self, site): site.announce(mode="update", pex=False) @@ -399,7 +379,7 @@ class FileServer(ConnectionServer): thread_reaload_tracker_files = gevent.spawn(self.reloadTrackerFilesThread) thread_announce_sites = gevent.spawn(self.announceSites) - thread_cleanup_sites = gevent.spawn(self.cleanupSites) + thread_sites_maintenance = gevent.spawn(self.sitesMaintenanceThread) thread_wakeup_watcher = gevent.spawn(self.wakeupWatcher) ConnectionServer.listen(self) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 75eb190f..0a518fdc 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -46,6 +46,7 @@ class Peer(object): self.is_tracker_connection = False # Tracker connection instead of normal peer self.reputation = 0 # More likely to connect if larger self.last_content_json_update = 0.0 # Modify date of last received content.json + self.protected = 0 self.connection_error = 0 # Series of connection error self.hash_failed = 0 # Number of bad files from peer @@ -74,9 +75,26 @@ class Peer(object): logger.log(self.log_level, "%s:%s %s" % (self.ip, self.port, text)) + # Site marks its Peers protected, if it has not enough peers connected. + # This is to be used to prevent disconnecting from peers when doing + # a periodic cleanup. + def markProtected(self, interval=60*20): + self.protected = time.time() + interval + + def isProtected(self): + if self.protected > 0: + if self.protected < time.time(): + self.protected = 0 + return self.protected > 0 + def isConnected(self): + if self.connection and not self.connection.connected: + self.connection = None return self.connection and self.connection.connected + def isTtlExpired(self, ttl): + return (time.time() - self.time_found) > ttl + # Connect to host def connect(self, connection=None): if self.reputation < -10: @@ -115,6 +133,11 @@ class Peer(object): self.connection = None return self.connection + def disconnect(self, reason="Unknown"): + if self.connection: + self.connection.close(reason) + self.connection = None + # Check if we have connection to peer def findConnection(self): if self.connection and self.connection.connected: # We have connection to peer @@ -400,8 +423,7 @@ class Peer(object): if self.site and self in self.site.peers_recent: self.site.peers_recent.remove(self) - if self.connection: - self.connection.close(reason) + self.disconnect(reason) # - EVENTS - diff --git a/src/Site/Site.py b/src/Site/Site.py index fff592cd..b941b035 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -40,6 +40,9 @@ class Site(object): self.log = logging.getLogger("Site:%s" % self.address_short) self.addEventListeners() + self.periodic_maintenance_interval = 60 * 20 + self.periodic_maintenance_timestamp = 0 + self.content = None # Load content.json self.peers = {} # Key: ip:port, Value: Peer.Peer self.peers_recent = collections.deque(maxlen=150) @@ -853,6 +856,11 @@ class Site(object): if self.isServing(): self.announcer.announce(*args, **kwargs) + # The engine tries to maintain the number of active connections: + # >= getPreferableActiveConnectionCount() + # and + # <= getActiveConnectionCountLimit() + def getPreferableActiveConnectionCount(self): if not self.isServing(): return 0 @@ -874,8 +882,16 @@ class Site(object): if len(self.peers) < 50: count = max(count, 5) + count = min(count, config.connected_limit) + return count + def getActiveConnectionCountLimit(self): + count_above_preferable = 2 + limit = self.getPreferableActiveConnectionCount() + count_above_preferable + limit = min(limit, config.connected_limit) + return limit + def tryConnectingToMorePeers(self, more=1, pex=True, try_harder=False): max_peers = more * 2 + 10 if try_harder: @@ -920,7 +936,7 @@ class Site(object): if num is None: num = self.getPreferableActiveConnectionCount() - need = min(len(self.peers), num, config.connected_limit) + need = min(len(self.peers), num) connected = self.bringConnections( need=need, @@ -935,8 +951,15 @@ class Site(object): pex=pex, try_harder=True) + if connected < num: + self.markConnectedPeersProtected() + return connected + def markConnectedPeersProtected(self): + for peer in site.getConnectedPeers(): + peer.markProtected() + # Return: Probably peers verified to be connectable recently def getConnectablePeers(self, need_num=5, ignore=[], allow_private=True): peers = list(self.peers.values()) @@ -1022,53 +1045,87 @@ class Site(object): back.append(peer) return back - # Cleanup probably dead peers and close connection if too much - def cleanupPeers(self, peers_protected=[]): + def removeDeadPeers(self): peers = list(self.peers.values()) - if len(peers) > 20: - # Cleanup old peers - removed = 0 - if len(peers) > 1000: - ttl = 60 * 60 * 1 - else: - ttl = 60 * 60 * 4 + if len(peers) <= 20: + return - for peer in peers: - if peer.connection and peer.connection.connected: - continue - if peer.connection and not peer.connection.connected: - peer.connection = None # Dead connection - if time.time() - peer.time_found > ttl: # Not found on tracker or via pex in last 4 hour - peer.remove("Time found expired") - removed += 1 - if removed > len(peers) * 0.1: # Don't remove too much at once - break + removed = 0 + if len(peers) > 1000: + ttl = 60 * 60 * 1 + elif len(peers) > 100: + ttl = 60 * 60 * 4 + else: + ttl = 60 * 60 * 8 - if removed: - self.log.debug("Cleanup peers result: Removed %s, left: %s" % (removed, len(self.peers))) + for peer in peers: + if peer.isConnected() or peer.isProtected(): + continue + if peer.isTtlExpired(ttl): + peer.remove("TTL expired") + removed += 1 + if removed > len(peers) * 0.1: # Don't remove too much at once + break - # Close peers over the limit - closed = 0 - connected_peers = [peer for peer in self.getConnectedPeers() if peer.connection.connected] # Only fully connected peers - need_to_close = len(connected_peers) - config.connected_limit + if removed: + self.log.debug("Cleanup peers result: Removed %s, left: %s" % (removed, len(self.peers))) - if closed < need_to_close: - # Try to keep connections with more sites + # Cleanup probably dead peers and close connection if too much + def cleanupPeers(self): + self.removeDeadPeers() + + limit = self.getActiveConnectionCountLimit() + connected_peers = [peer for peer in self.getConnectedPeers() if peer.isConnected()] # Only fully connected peers + need_to_close = len(connected_peers) - limit + + if need_to_close > 0: + closed = 0 for peer in sorted(connected_peers, key=lambda peer: min(peer.connection.sites, 5)): - if not peer.connection: + if not peer.isConnected(): continue - if peer.key in peers_protected: + if peer.isProtected(): continue if peer.connection.sites > 5: break - peer.connection.close("Cleanup peers") - peer.connection = None + peer.disconnect("Cleanup peers") closed += 1 if closed >= need_to_close: break - if need_to_close > 0: - self.log.debug("Connected: %s, Need to close: %s, Closed: %s" % (len(connected_peers), need_to_close, closed)) + self.log.debug("Connected: %s, Need to close: %s, Closed: %s" % ( + len(connected_peers), need_to_close, closed)) + + def runPeriodicMaintenance(self, startup=False, force=False): + if not self.isServing(): + return False + + scheduled_time = self.periodic_maintenance_timestamp + self.periodic_maintenance_interval + + if time.time() < scheduled_time and not force: + return False + + self.periodic_maintenance_timestamp = time.time() + + self.log.debug("runPeriodicMaintenance: startup=%s" % startup) + + if not startup: + self.cleanupPeers() + + if self.peers: + with gevent.Timeout(10, exception=False): + self.announcer.announcePex() + + # Last check modification failed + if self.content_updated is False: + self.update() + elif self.bad_files: + self.retryBadFiles() + + self.needConnections(check_site_on_reconnect=True) + + self.periodic_maintenance_timestamp = time.time() + + return True # Send hashfield to peers def sendMyHashfield(self, limit=5): diff --git a/src/util/CircularIterator.py b/src/util/CircularIterator.py new file mode 100644 index 00000000..3466092e --- /dev/null +++ b/src/util/CircularIterator.py @@ -0,0 +1,34 @@ +import random + +class CircularIterator: + def __init__(self): + self.successive_count = 0 + self.last_size = 0 + self.index = -1 + + def next(self, items): + self.last_size = len(items) + + if self.last_size == 0: + return None + + if self.index < 0: + self.index = random.randint(0, self.last_size) + else: + self.index += 1 + + self.index = self.index % self.last_size + + self.successive_count += 1 + + return items[self.index] + + def resetSuccessiveCount(self): + self.successive_count = 0 + + def getSuccessiveCount(self): + return self.successive_count + + def isWrapped(self): + return self.successive_count >= self.last_size + diff --git a/src/util/__init__.py b/src/util/__init__.py index ab8a8b88..f00c1459 100644 --- a/src/util/__init__.py +++ b/src/util/__init__.py @@ -1,4 +1,5 @@ from .Cached import Cached +from .CircularIterator import CircularIterator from .Event import Event from .Noparallel import Noparallel from .Pooled import Pooled From adf40dbb6b24234bc72cec77710632ceae52cbd6 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 29 Oct 2020 12:57:16 +0700 Subject: [PATCH 038/114] Refactor SiteAnnouncer.announce --- src/Site/SiteAnnouncer.py | 193 ++++++++++++++++++++++---------------- 1 file changed, 114 insertions(+), 79 deletions(-) diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index b50a01fe..1440eb57 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -13,6 +13,7 @@ from Debug import Debug from util import helper from greenlet import GreenletExit import util +from util import CircularIterator class AnnounceError(Exception): @@ -30,7 +31,7 @@ class SiteAnnouncer(object): self.stats = {} self.fileserver_port = config.fileserver_port self.peer_id = self.site.connection_server.peer_id - self.last_tracker_id = random.randint(0, 10) + self.tracker_circular_iterator = CircularIterator() self.time_last_announce = 0 def getTrackers(self): @@ -49,15 +50,58 @@ class SiteAnnouncer(object): return trackers - def getAnnouncingTrackers(self, mode): + def shouldTrackerBeTemporarilyIgnored(self, tracker, mode, force): + if not tracker: + return True + + if force: + return False + + now = time.time() + + # Throttle accessing unresponsive trackers + tracker_stats = global_stats[tracker] + delay = min(30 * tracker_stats["num_error"], 60 * 10) + time_announce_allowed = tracker_stats["time_request"] + delay + if now < time_announce_allowed: + return True + + return False + + def getAnnouncingTrackers(self, mode, force): trackers = self.getSupportedTrackers() - if trackers and (mode == "update" or mode == "more"): # Only announce on one tracker, increment the queried tracker id - self.last_tracker_id += 1 - self.last_tracker_id = self.last_tracker_id % len(trackers) - trackers_announcing = [trackers[self.last_tracker_id]] # We only going to use this one + if trackers and (mode == "update" or mode == "more"): + + # Choose just 2 trackers to announce to + + trackers_announcing = [] + + # One is the next in sequence + + self.tracker_circular_iterator.resetSuccessiveCount() + while 1: + tracker = self.tracker_circular_iterator.next(trackers) + if not self.shouldTrackerBeTemporarilyIgnored(tracker, mode, force): + trackers_announcing.append(tracker) + break + if self.tracker_circular_iterator.isWrapped(): + break + + # And one is just random + + shuffled_trackers = random.sample(trackers, len(trackers)) + for tracker in shuffled_trackers: + if tracker in trackers_announcing: + continue + if not self.shouldTrackerBeTemporarilyIgnored(tracker, mode, force): + trackers_announcing.append(tracker) + break else: - trackers_announcing = trackers + trackers_announcing = [ + tracker for tracker in trackers + if not self.shouldTrackerBeTemporarilyIgnored(tracker, mode, force) + ] return trackers_announcing @@ -76,83 +120,18 @@ class SiteAnnouncer(object): def announce(self, force=False, mode="start", pex=True): if time.time() - self.time_last_announce < 30 and not force: return # No reannouncing within 30 secs - if force: - self.log.debug("Force reannounce in mode %s" % mode) + + self.log.debug("announce: force=%s, mode=%s, pex=%s" % (force, mode, pex)) self.fileserver_port = config.fileserver_port self.time_last_announce = time.time() - trackers = self.getAnnouncingTrackers(mode) - - if config.verbose: - self.log.debug("Tracker announcing, trackers: %s" % trackers) - - errors = [] - slow = [] - s = time.time() - threads = [] - num_announced = 0 - - for tracker in trackers: # Start announce threads - tracker_stats = global_stats[tracker] - # Reduce the announce time for trackers that looks unreliable - time_announce_allowed = time.time() - 60 * min(30, tracker_stats["num_error"]) - if tracker_stats["num_error"] > 5 and tracker_stats["time_request"] > time_announce_allowed and not force: - if config.verbose: - self.log.debug("Tracker %s looks unreliable, announce skipped (error: %s)" % (tracker, tracker_stats["num_error"])) - continue - thread = self.site.greenlet_manager.spawn(self.announceTracker, tracker, mode=mode) - threads.append(thread) - thread.tracker = tracker - - time.sleep(0.01) - self.updateWebsocket(trackers="announcing") - - gevent.joinall(threads, timeout=20) # Wait for announce finish - - for thread in threads: - if thread.value is None: - continue - if thread.value is not False: - if thread.value > 1.0: # Takes more than 1 second to announce - slow.append("%.2fs %s" % (thread.value, thread.tracker)) - num_announced += 1 - else: - if thread.ready(): - errors.append(thread.tracker) - else: # Still running - slow.append("30s+ %s" % thread.tracker) - - # Save peers num - self.site.settings["peers"] = len(self.site.peers) - - if len(errors) < len(threads): # At least one tracker finished - if len(trackers) == 1: - announced_to = trackers[0] - else: - announced_to = "%s/%s trackers" % (num_announced, len(threads)) - if mode != "update" or config.verbose: - self.log.debug( - "Announced in mode %s to %s in %.3fs, errors: %s, slow: %s" % - (mode, announced_to, time.time() - s, errors, slow) - ) - else: - if len(threads) > 1: - self.log.error("Announce to %s trackers in %.3fs, failed" % (len(threads), time.time() - s)) - if len(threads) == 1 and mode != "start": # Move to next tracker - self.log.debug("Tracker failed, skipping to next one...") - self.site.greenlet_manager.spawnLater(1.0, self.announce, force=force, mode=mode, pex=pex) - - self.updateWebsocket(trackers="announced") + trackers = self.getAnnouncingTrackers(mode, force) + self.log.debug("Chosen trackers: %s" % trackers) + self.announceToTrackers(trackers, force=force, mode=mode) if pex: - self.updateWebsocket(pex="announcing") - if mode == "more": # Need more peers - self.announcePex(need_num=10) - else: - self.announcePex() - - self.updateWebsocket(pex="announced") + self.announcePex() def getTrackerHandler(self, protocol): return None @@ -258,8 +237,62 @@ class SiteAnnouncer(object): ) return time.time() - s + def announceToTrackers(self, trackers, force=False, mode="start"): + errors = [] + slow = [] + s = time.time() + threads = [] + num_announced = 0 + + for tracker in trackers: # Start announce threads + thread = self.site.greenlet_manager.spawn(self.announceTracker, tracker, mode=mode) + threads.append(thread) + thread.tracker = tracker + + time.sleep(0.01) + self.updateWebsocket(trackers="announcing") + + gevent.joinall(threads, timeout=20) # Wait for announce finish + + for thread in threads: + if thread.value is None: + continue + if thread.value is not False: + if thread.value > 1.0: # Takes more than 1 second to announce + slow.append("%.2fs %s" % (thread.value, thread.tracker)) + num_announced += 1 + else: + if thread.ready(): + errors.append(thread.tracker) + else: # Still running + slow.append("30s+ %s" % thread.tracker) + + # Save peers num + self.site.settings["peers"] = len(self.site.peers) + + if len(errors) < len(threads): # At least one tracker finished + if len(trackers) == 1: + announced_to = trackers[0] + else: + announced_to = "%s/%s trackers" % (num_announced, len(threads)) + if mode != "update" or config.verbose: + self.log.debug( + "Announced in mode %s to %s in %.3fs, errors: %s, slow: %s" % + (mode, announced_to, time.time() - s, errors, slow) + ) + else: + if len(threads) > 1: + self.log.error("Announce to %s trackers in %.3fs, failed" % (len(threads), time.time() - s)) + if len(threads) > 1 and mode != "start": # Move to next tracker + self.log.debug("Tracker failed, skipping to next one...") + self.site.greenlet_manager.spawnLater(5.0, self.announce, force=force, mode=mode, pex=False) + + self.updateWebsocket(trackers="announced") + @util.Noparallel(blocking=False) - def announcePex(self, query_num=2, need_num=5): + def announcePex(self, query_num=2, need_num=10): + self.updateWebsocket(pex="announcing") + peers = self.site.getConnectedPeers() if len(peers) == 0: # Wait 3s for connections time.sleep(3) @@ -286,6 +319,8 @@ class SiteAnnouncer(object): break self.log.debug("Pex result: from %s peers got %s new peers." % (done, total_added)) + self.updateWebsocket(pex="announced") + def updateWebsocket(self, **kwargs): if kwargs: param = {"event": list(kwargs.items())[0]} From 829fd4678133e527d34ac395c6c5bf3da20f8050 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 30 Oct 2020 14:36:08 +0700 Subject: [PATCH 039/114] Redesign the site updating strategy in Site.py, SiteAnnouncer.py, FileServer.py --- src/Config.py | 2 + src/File/FileServer.py | 245 ++++++++++++++++++++++---------------- src/Site/Site.py | 156 ++++++++++++++++++++---- src/Site/SiteAnnouncer.py | 11 +- 4 files changed, 285 insertions(+), 129 deletions(-) diff --git a/src/Config.py b/src/Config.py index 70b284bf..a3d66395 100644 --- a/src/Config.py +++ b/src/Config.py @@ -257,6 +257,8 @@ class Config(object): self.parser.add_argument('--global_connected_limit', help='Max connections', default=512, type=int, metavar='global_connected_limit') self.parser.add_argument('--workers', help='Download workers per site', default=5, type=int, metavar='workers') + self.parser.add_argument('--expose_no_ownership', help='By default, ZeroNet tries checking updates for own sites more frequently. This can be used by a third party for revealing the network addresses of a site owner. If this option is enabled, ZeroNet performs the checks in the same way for any sites.', type='bool', choices=[True, False], default=False) + self.parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') self.parser.add_argument('--fileserver_port', help='FileServer bind port (0: randomize)', default=0, type=int, metavar='port') self.parser.add_argument('--fileserver_port_range', help='FileServer randomization range', default="10000-40000", metavar='port') diff --git a/src/File/FileServer.py b/src/File/FileServer.py index fbc3cc85..8294d179 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -10,7 +10,6 @@ from gevent.server import StreamServer import util from util import helper -from util import CircularIterator from Config import config from .FileRequest import FileRequest from Peer import PeerPortchecker @@ -19,16 +18,24 @@ from Connection import ConnectionServer from Plugin import PluginManager from Debug import Debug +log = logging.getLogger("FileServer") + @PluginManager.acceptPlugins class FileServer(ConnectionServer): def __init__(self, ip=config.fileserver_ip, port=config.fileserver_port, ip_type=config.fileserver_ip_type): self.site_manager = SiteManager.site_manager self.portchecker = PeerPortchecker.PeerPortchecker(self) - self.log = logging.getLogger("FileServer") self.ip_type = ip_type self.ip_external_list = [] + # This is wrong: + # self.log = logging.getLogger("FileServer") + # The value of self.log will be overwritten in ConnectionServer.__init__() + + self.check_pool = gevent.pool.Pool(5) + self.check_start_time = 0 + self.supported_ip_types = ["ipv4"] # Outgoing ip_type support if helper.getIpType(ip) == "ipv6" or self.isIpv6Supported(): self.supported_ip_types.append("ipv6") @@ -52,17 +59,17 @@ class FileServer(ConnectionServer): config.arguments.fileserver_port = port ConnectionServer.__init__(self, ip, port, self.handleRequest) - self.log.debug("Supported IP types: %s" % self.supported_ip_types) + log.debug("Supported IP types: %s" % self.supported_ip_types) if ip_type == "dual" and ip == "::": # Also bind to ipv4 addres in dual mode try: - self.log.debug("Binding proxy to %s:%s" % ("::", self.port)) + log.debug("Binding proxy to %s:%s" % ("::", self.port)) self.stream_server_proxy = StreamServer( ("0.0.0.0", self.port), self.handleIncomingConnection, spawn=self.pool, backlog=100 ) except Exception as err: - self.log.info("StreamServer proxy create error: %s" % Debug.formatException(err)) + log.info("StreamServer proxy create error: %s" % Debug.formatException(err)) self.port_opened = {} @@ -71,8 +78,17 @@ class FileServer(ConnectionServer): self.files_parsing = {} self.ui_server = None + def getSiteAddresses(self): + # Avoid saving the site list on the stack, since a site may be deleted + # from the original list while iterating. + # Use the list of addresses instead. + return [ + site.address for site in + sorted(list(self.sites.values()), key=lambda site: site.settings.get("modified", 0), reverse=True) + ] + def getRandomPort(self, ip, port_range_from, port_range_to): - self.log.info("Getting random port in range %s-%s..." % (port_range_from, port_range_to)) + log.info("Getting random port in range %s-%s..." % (port_range_from, port_range_to)) tried = [] for bind_retry in range(100): port = random.randint(port_range_from, port_range_to) @@ -84,11 +100,11 @@ class FileServer(ConnectionServer): sock.bind((ip, port)) success = True except Exception as err: - self.log.warning("Error binding to port %s: %s" % (port, err)) + log.warning("Error binding to port %s: %s" % (port, err)) success = False sock.close() if success: - self.log.info("Found unused random port: %s" % port) + log.info("Found unused random port: %s" % port) return port else: time.sleep(0.1) @@ -104,16 +120,16 @@ class FileServer(ConnectionServer): sock.connect((ipv6_testip, 80)) local_ipv6 = sock.getsockname()[0] if local_ipv6 == "::1": - self.log.debug("IPv6 not supported, no local IPv6 address") + log.debug("IPv6 not supported, no local IPv6 address") return False else: - self.log.debug("IPv6 supported on IP %s" % local_ipv6) + log.debug("IPv6 supported on IP %s" % local_ipv6) return True except socket.error as err: - self.log.warning("IPv6 not supported: %s" % err) + log.warning("IPv6 not supported: %s" % err) return False except Exception as err: - self.log.error("IPv6 check error: %s" % err) + log.error("IPv6 check error: %s" % err) return False def listenProxy(self): @@ -121,20 +137,20 @@ class FileServer(ConnectionServer): self.stream_server_proxy.serve_forever() except Exception as err: if err.errno == 98: # Address already in use error - self.log.debug("StreamServer proxy listen error: %s" % err) + log.debug("StreamServer proxy listen error: %s" % err) else: - self.log.info("StreamServer proxy listen error: %s" % err) + log.info("StreamServer proxy listen error: %s" % err) # Handle request to fileserver def handleRequest(self, connection, message): if config.verbose: if "params" in message: - self.log.debug( + log.debug( "FileRequest: %s %s %s %s" % (str(connection), message["cmd"], message["params"].get("site"), message["params"].get("inner_path")) ) else: - self.log.debug("FileRequest: %s %s" % (str(connection), message["cmd"])) + log.debug("FileRequest: %s %s" % (str(connection), message["cmd"])) req = FileRequest(self, connection) req.route(message["cmd"], message.get("req_id"), message.get("params")) if not self.has_internet and not connection.is_private_ip: @@ -142,7 +158,7 @@ class FileServer(ConnectionServer): self.onInternetOnline() def onInternetOnline(self): - self.log.info("Internet online") + log.info("Internet online") gevent.spawn(self.checkSites, check_files=False, force_port_check=True) # Reload the FileRequest class to prevent restarts in debug mode @@ -153,7 +169,7 @@ class FileServer(ConnectionServer): def portCheck(self): if config.offline: - self.log.info("Offline mode: port check disabled") + log.info("Offline mode: port check disabled") res = {"ipv4": None, "ipv6": None} self.port_opened = res return res @@ -169,7 +185,7 @@ class FileServer(ConnectionServer): } self.ip_external_list = config.ip_external self.port_opened.update(res) - self.log.info("Server port opened based on configuration ipv4: %s, ipv6: %s" % (res["ipv4"], res["ipv6"])) + log.info("Server port opened based on configuration ipv4: %s, ipv6: %s" % (res["ipv4"], res["ipv6"])) return res self.port_opened = {} @@ -191,7 +207,7 @@ class FileServer(ConnectionServer): else: res_ipv6 = res_ipv6_thread.get() if res_ipv6["opened"] and not helper.getIpType(res_ipv6["ip"]) == "ipv6": - self.log.info("Invalid IPv6 address from port check: %s" % res_ipv6["ip"]) + log.info("Invalid IPv6 address from port check: %s" % res_ipv6["ip"]) res_ipv6["opened"] = False self.ip_external_list = [] @@ -200,7 +216,7 @@ class FileServer(ConnectionServer): self.ip_external_list.append(res_ip["ip"]) SiteManager.peer_blacklist.append((res_ip["ip"], self.port)) - self.log.info("Server port opened ipv4: %s, ipv6: %s" % (res_ipv4["opened"], res_ipv6["opened"])) + log.info("Server port opened ipv4: %s, ipv6: %s" % (res_ipv4["opened"], res_ipv6["opened"])) res = {"ipv4": res_ipv4["opened"], "ipv6": res_ipv6["opened"]} @@ -213,7 +229,7 @@ class FileServer(ConnectionServer): self.ip_external_list.append(ip) res[helper.getIpType(ip)] = True # We have opened port if we have external ip SiteManager.peer_blacklist.append((ip, self.port)) - self.log.debug("External ip found on interfaces: %s" % ip) + log.debug("External ip found on interfaces: %s" % ip) self.port_opened.update(res) @@ -224,79 +240,123 @@ class FileServer(ConnectionServer): # Check site file integrity def checkSite(self, site, check_files=False): - if site.isServing(): - site.announce(mode="startup") # Announce site to tracker - site.update(check_files=check_files) # Update site's content.json and download changed files - site.sendMyHashfield() - site.updateHashfield() + if not site.isServing(): + return + + quick_start = len(site.peers) >= 50 + + if quick_start: + log.debug("Checking site: %s (quick start)" % (site.address)) + else: + log.debug("Checking site: %s" % (site.address)) + + if quick_start: + site.setDelayedStartupAnnounce() + else: + site.announce(mode="startup") + + site.update(check_files=check_files) # Update site's content.json and download changed files + site.sendMyHashfield() + site.updateHashfield() # Check sites integrity @util.Noparallel() def checkSites(self, check_files=False, force_port_check=False): - self.log.debug("Checking sites...") - s = time.time() - sites_checking = False - if not self.port_opened or force_port_check: # Test and open port if not tested yet - if len(self.sites) <= 2: # Don't wait port opening on first startup - sites_checking = True - for address, site in list(self.sites.items()): - gevent.spawn(self.checkSite, site, check_files) + log.info("Checking sites: check_files=%s, force_port_check=%s", check_files, force_port_check) + # Don't wait port opening on first startup. Do the instant check now. + if len(self.sites) <= 2: + sites_checking = True + for address, site in list(self.sites.items()): + gevent.spawn(self.checkSite, site, check_files) + + # Test and open port if not tested yet + if not self.port_opened or force_port_check: self.portCheck() - if not self.port_opened["ipv4"]: self.tor_manager.startOnions() - if not sites_checking: - check_pool = gevent.pool.Pool(5) - # Check sites integrity - for site in sorted(list(self.sites.values()), key=lambda site: site.settings.get("modified", 0), reverse=True): - if not site.isServing(): - continue - check_thread = check_pool.spawn(self.checkSite, site, check_files) # Check in new thread - time.sleep(2) - if site.settings.get("modified", 0) < time.time() - 60 * 60 * 24: # Not so active site, wait some sec to finish - check_thread.join(timeout=5) - self.log.debug("Checksites done in %.3fs" % (time.time() - s)) + site_addresses = self.getSiteAddresses() + + sites_processed = 0 + start_time = time.time() + self.check_start_time = start_time + progress_print_time = time.time() + + # Check sites integrity + for site_address in site_addresses: + site = self.sites.get(site_address, None) + + sites_processed += 1 + + if (not site) or (not site.isServing()): + continue + check_thread = self.check_pool.spawn(self.checkSite, site, check_files) # Check in new thread + + if time.time() - progress_print_time > 60: + progress_print_time = time.time() + time_spent = time.time() - start_time + time_per_site = time_spent / float(sites_processed) + sites_left = len(site_addresses) - sites_processed + time_left = time_per_site * sites_left + log.info("Checking sites: DONE: %d sites in %.2fs (%.2fs per site); LEFT: %d sites in %.2fs", + sites_processed, + time_spent, + time_per_site, + sites_left, + time_left + ) + + if (self.check_start_time != start_time) and self.check_pool.full(): + # Another check is running, throttling... + time.sleep(5) + else: + time.sleep(1) + + log.info("Checking sites: finished in %.3fs" % (time.time() - start_time)) def sitesMaintenanceThread(self): import gc startup = True short_timeout = 2 - long_timeout = 60 * 5 - - circular_iterator = CircularIterator() + long_timeout = 60 * 2 while 1: - if circular_iterator.isWrapped(): - time.sleep(long_timeout) - circular_iterator.resetSuccessiveCount() - gc.collect() # Explicit garbage collection + time.sleep(long_timeout) + gc.collect() # Explicit garbage collection - self.log.debug( - "Running site cleanup, connections: %s, internet: %s" % - (len(self.connections), self.has_internet) + log.debug( + "Starting maintenance cycle: connections=%s, internet=%s", + len(self.connections), self.has_internet + ) + start_time = time.time() + + site_addresses = self.getSiteAddresses() + + sites_processed = 0 + + for site_address in site_addresses: + site = self.sites.get(site_address, None) + if (not site) or (not site.isServing()): + continue + + log.debug("Running maintenance for site: %s", site.address) + + done = site.runPeriodicMaintenance(startup=startup) + site = None + if done: + sites_processed += 1 + time.sleep(short_timeout) + + log.debug("Maintenance cycle finished in %.3fs. Total sites: %d. Processed sites: %d", + time.time() - start_time, + len(site_addresses), + sites_processed ) - site = circular_iterator.next(list(self.sites.values())) - if site: - done = site.runPeriodicMaintenance(startup=startup) - if done: - time.sleep(short_timeout) - site = None - - if circular_iterator.isWrapped(): - startup = False - - def announceSite(self, site): - site.announce(mode="update", pex=False) - active_site = time.time() - site.settings.get("modified", 0) < 24 * 60 * 60 - if site.settings["own"] or active_site: - # Check connections more frequently on own and active sites to speed-up first connections - site.needConnections(check_site_on_reconnect=True) - site.sendMyHashfield(3) - site.updateHashfield(3) + site_addresses = None + startup = False # Periodic reloading of tracker files def reloadTrackerFilesThread(self): @@ -309,24 +369,6 @@ class FileServer(ConnectionServer): time.sleep(interval) config.loadTrackersFile() - # Announce sites every 20 min - def announceSites(self): - time.sleep(5 * 60) # Sites already announced on startup - while 1: - s = time.time() - for address, site in list(self.sites.items()): - if not site.isServing(): - continue - gevent.spawn(self.announceSite, site).join(timeout=10) - time.sleep(1) - taken = time.time() - s - - # Query all trackers one-by-one in 20 minutes evenly distributed - sleep = max(0, 60 * 20 / len(config.trackers) - taken) - - self.log.debug("Site announce tracker done in %.3fs, sleeping for %.3fs..." % (taken, sleep)) - time.sleep(sleep) - # Detects if computer back from wakeup def wakeupWatcher(self): last_time = time.time() @@ -336,7 +378,7 @@ class FileServer(ConnectionServer): is_time_changed = time.time() - max(self.last_request, last_time) > 60 * 3 if is_time_changed: # If taken more than 3 minute then the computer was in sleep mode - self.log.info( + log.info( "Wakeup detected: time warp from %0.f to %0.f (%0.f sleep seconds), acting like startup..." % (last_time, time.time(), time.time() - last_time) ) @@ -344,7 +386,7 @@ class FileServer(ConnectionServer): my_ips = socket.gethostbyname_ex('')[2] is_ip_changed = my_ips != last_my_ips if is_ip_changed: - self.log.info("IP change detected from %s to %s" % (last_my_ips, my_ips)) + log.info("IP change detected from %s to %s" % (last_my_ips, my_ips)) if is_time_changed or is_ip_changed: self.checkSites(check_files=False, force_port_check=True) @@ -362,9 +404,9 @@ class FileServer(ConnectionServer): try: self.stream_server.start() except Exception as err: - self.log.error("Error listening on: %s:%s: %s" % (self.ip, self.port, err)) + log.error("Error listening on: %s:%s: %s" % (self.ip, self.port, err)) if "ui_server" in dir(sys.modules["main"]): - self.log.debug("Stopping UI Server.") + log.debug("Stopping UI Server.") sys.modules["main"].ui_server.stop() return False @@ -378,21 +420,20 @@ class FileServer(ConnectionServer): gevent.spawn(self.checkSites) thread_reaload_tracker_files = gevent.spawn(self.reloadTrackerFilesThread) - thread_announce_sites = gevent.spawn(self.announceSites) thread_sites_maintenance = gevent.spawn(self.sitesMaintenanceThread) thread_wakeup_watcher = gevent.spawn(self.wakeupWatcher) ConnectionServer.listen(self) - self.log.debug("Stopped.") + log.debug("Stopped.") def stop(self): if self.running and self.portchecker.upnp_port_opened: - self.log.debug('Closing port %d' % self.port) + log.debug('Closing port %d' % self.port) try: self.portchecker.portClose(self.port) - self.log.info('Closed port via upnp.') + log.info('Closed port via upnp.') except Exception as err: - self.log.info("Failed at attempt to use upnp to close port: %s" % err) + log.info("Failed at attempt to use upnp to close port: %s" % err) return ConnectionServer.stop(self) diff --git a/src/Site/Site.py b/src/Site/Site.py index b941b035..b01e9aae 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -28,6 +28,38 @@ from File import FileServer from .SiteAnnouncer import SiteAnnouncer from . import SiteManager +class ScaledTimeoutHandler: + def __init__(self, val_min, val_max, handler=None, scaler=None): + self.val_min = val_min + self.val_max = val_max + self.timestamp = 0 + self.handler = handler + self.scaler = scaler + self.log = logging.getLogger("ScaledTimeoutHandler") + + def isExpired(self, scale): + interval = scale * (self.val_max - self.val_min) + self.val_min + expired_at = self.timestamp + interval + now = time.time() + expired = (now > expired_at) + if expired: + self.log.debug( + "Expired: [%d..%d]: scale=%f, interval=%f", + self.val_min, self.val_max, scale, interval) + return expired + + def done(self): + self.timestamp = time.time() + + def run(self, *args, **kwargs): + do_run = kwargs["force"] or self.isExpired(self.scaler()) + if do_run: + result = self.handler(*args, **kwargs) + if result: + self.done() + return result + else: + return None @PluginManager.acceptPlugins class Site(object): @@ -40,8 +72,16 @@ class Site(object): self.log = logging.getLogger("Site:%s" % self.address_short) self.addEventListeners() - self.periodic_maintenance_interval = 60 * 20 - self.periodic_maintenance_timestamp = 0 + self.periodic_maintenance_handlers = [ + ScaledTimeoutHandler(60 * 30, 60 * 2, + handler=self.periodicMaintenanceHandler_announce, + scaler=self.getAnnounceRating), + ScaledTimeoutHandler(60 * 20, 60 * 10, + handler=self.periodicMaintenanceHandler_general, + scaler=self.getActivityRating) + ] + + self.delayed_startup_announce = False self.content = None # Load content.json self.peers = {} # Key: ip:port, Value: Peer.Peer @@ -852,10 +892,63 @@ class Site(object): peer.found(source) return peer + def setDelayedStartupAnnounce(self): + self.delayed_startup_announce = True + + def applyDelayedStartupAnnounce(self): + if self.delayed_startup_announce: + self.delayed_startup_announce = False + self.announce(mode="startup") + return True + return False + def announce(self, *args, **kwargs): if self.isServing(): self.announcer.announce(*args, **kwargs) + def getActivityRating(self, force_safe=False): + age = time.time() - self.settings.get("modified", 0) + + if age < 60 * 60: + rating = 1.0 + elif age < 60 * 60 * 5: + rating = 0.8 + elif age < 60 * 60 * 24: + rating = 0.6 + elif age < 60 * 60 * 24 * 3: + rating = 0.4 + elif age < 60 * 60 * 24 * 7: + rating = 0.2 + else: + rating = 0.0 + + force_safe = force_safe or config.expose_no_ownership + + if (not force_safe) and self.settings["own"]: + rating = min(rating, 0.6) + + return rating + + def getAnnounceRating(self): + # rare frequent + # announces announces + # 0 ------------------- 1 + # activity -------------> -- active site ==> frequent announces + # <---------------- peers -- many peers ==> rare announces + # trackers -------------> -- many trackers ==> frequent announces to iterate over more trackers + + activity_rating = self.getActivityRating(force_safe=True) + + peer_count = len(self.peers) + peer_rating = 1.0 - min(peer_count, 50) / 50.0 + + tracker_count = self.announcer.getSupportedTrackerCount() + tracker_count = max(tracker_count, 1) + tracker_rating = 1.0 - (1.0 / tracker_count) + + v = [activity_rating, peer_rating, tracker_rating] + return sum(v) / float(len(v)) + # The engine tries to maintain the number of active connections: # >= getPreferableActiveConnectionCount() # and @@ -866,18 +959,7 @@ class Site(object): return 0 age = time.time() - self.settings.get("modified", 0) - count = 0 - - if age < 60 * 60: - count = 10 - elif age < 60 * 60 * 5: - count = 8 - elif age < 60 * 60 * 24: - count = 6 - elif age < 60 * 60 * 24 * 3: - count = 4 - elif age < 60 * 60 * 24 * 7: - count = 2 + count = int(10 * self.getActivityRating(force_safe=True)) if len(self.peers) < 50: count = max(count, 5) @@ -957,7 +1039,7 @@ class Site(object): return connected def markConnectedPeersProtected(self): - for peer in site.getConnectedPeers(): + for peer in self.getConnectedPeers(): peer.markProtected() # Return: Probably peers verified to be connectable recently @@ -1099,32 +1181,54 @@ class Site(object): if not self.isServing(): return False - scheduled_time = self.periodic_maintenance_timestamp + self.periodic_maintenance_interval + self.log.debug("runPeriodicMaintenance: startup=%s, force=%s" % (startup, force)) - if time.time() < scheduled_time and not force: + result = False + + for handler in self.periodic_maintenance_handlers: + result = result | bool(handler.run(startup=startup, force=force)) + + return result + + def periodicMaintenanceHandler_general(self, startup=False, force=False): + if not self.isServing(): return False - self.periodic_maintenance_timestamp = time.time() + self.applyDelayedStartupAnnounce() - self.log.debug("runPeriodicMaintenance: startup=%s" % startup) + if not self.peers: + return False + + self.log.debug("periodicMaintenanceHandler_general: startup=%s, force=%s" % (startup, force)) if not startup: self.cleanupPeers() - if self.peers: - with gevent.Timeout(10, exception=False): - self.announcer.announcePex() + self.needConnections(check_site_on_reconnect=True) - # Last check modification failed - if self.content_updated is False: + with gevent.Timeout(10, exception=False): + self.announcer.announcePex() + + self.sendMyHashfield(3) + self.updateHashfield(3) + + if self.content_updated is False: # Last check modification failed self.update() elif self.bad_files: self.retryBadFiles() - self.needConnections(check_site_on_reconnect=True) + return True - self.periodic_maintenance_timestamp = time.time() + def periodicMaintenanceHandler_announce(self, startup=False, force=False): + if not self.isServing(): + return False + self.log.debug("periodicMaintenanceHandler_announce: startup=%s, force=%s" % (startup, force)) + + if self.applyDelayedStartupAnnounce(): + return True + + self.announce(mode="update", pex=False) return True # Send hashfield to peers diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index 1440eb57..5a97807e 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -33,6 +33,7 @@ class SiteAnnouncer(object): self.peer_id = self.site.connection_server.peer_id self.tracker_circular_iterator = CircularIterator() self.time_last_announce = 0 + self.supported_tracker_count = 0 def getTrackers(self): return config.trackers @@ -50,6 +51,12 @@ class SiteAnnouncer(object): return trackers + # Returns a cached value of len(self.getSupportedTrackers()), which can be + # inacurate. + # To be used from Site for estimating available tracker count. + def getSupportedTrackerCount(self): + return self.supported_tracker_count + def shouldTrackerBeTemporarilyIgnored(self, tracker, mode, force): if not tracker: return True @@ -71,6 +78,8 @@ class SiteAnnouncer(object): def getAnnouncingTrackers(self, mode, force): trackers = self.getSupportedTrackers() + self.supported_tracker_count = len(trackers) + if trackers and (mode == "update" or mode == "more"): # Choose just 2 trackers to announce to @@ -116,7 +125,7 @@ class SiteAnnouncer(object): back.append("onion") return back - @util.Noparallel(blocking=False) + @util.Noparallel() def announce(self, force=False, mode="start", pex=True): if time.time() - self.time_last_announce < 30 and not force: return # No reannouncing within 30 secs From d1b9cc826153248f4340284f531ab4258aa927e1 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 30 Oct 2020 23:28:16 +0700 Subject: [PATCH 040/114] Redesign the Internet outage detection. Improvements in FileServer threads. --- src/Connection/Connection.py | 12 +++ src/Connection/ConnectionServer.py | 65 +++++++++++---- src/File/FileServer.py | 125 +++++++++++++++++++++++++---- src/Site/Site.py | 1 + 4 files changed, 173 insertions(+), 30 deletions(-) diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 22bcf29c..27ae3734 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -442,6 +442,7 @@ class Connection(object): def handleMessage(self, message): cmd = message["cmd"] + self.updateOnlineStatus(successful_activity=True) self.last_message_time = time.time() self.last_cmd_recv = cmd if cmd == "response": # New style response @@ -504,6 +505,7 @@ class Connection(object): # Send data to connection def send(self, message, streaming=False): + self.updateOnlineStatus(outgoing_activity=True) self.last_send_time = time.time() if config.debug_socket: self.log("Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % ( @@ -543,6 +545,11 @@ class Connection(object): message = None with self.send_lock: self.sock.sendall(data) + # XXX: Should not be used here: + # self.updateOnlineStatus(successful_activity=True) + # Looks like self.sock.sendall() returns normally, instead of + # raising an Exception (at least, some times). + # So the only way of detecting the network activity is self.handleMessage() except Exception as err: self.close("Send error: %s (cmd: %s)" % (err, stat_key)) return False @@ -633,3 +640,8 @@ class Connection(object): self.sock = None self.unpacker = None self.event_connected = None + + def updateOnlineStatus(self, outgoing_activity=False, successful_activity=False): + self.server.updateOnlineStatus(self, + outgoing_activity=outgoing_activity, + successful_activity=successful_activity) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 8d377aca..66b50608 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -41,7 +41,11 @@ class ConnectionServer(object): self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood self.broken_ssl_ips = {} # Peerids of broken ssl connections self.ips = {} # Connection by ip + self.has_internet = True # Internet outage detection + self.last_outgoing_internet_activity_time = 0 # Last time the application tried to send any data + self.last_successful_internet_activity_time = 0 # Last time the application successfully sent or received any data + self.internet_outage_threshold = 60 * 2 self.stream_server = None self.stream_server_proxy = None @@ -60,6 +64,8 @@ class ConnectionServer(object): self.num_outgoing = 0 self.had_external_incoming = False + + self.timecorrection = 0.0 self.pool = Pool(500) # do not accept more than 500 connections @@ -252,8 +258,8 @@ class ConnectionServer(object): while self.running: run_i += 1 self.ip_incoming = {} # Reset connected ips counter - last_message_time = 0 s = time.time() + self.updateOnlineStatus(None) for connection in self.connections[:]: # Make a copy if connection.ip.endswith(".onion") or config.tor == "always": timeout_multipler = 2 @@ -261,9 +267,6 @@ class ConnectionServer(object): timeout_multipler = 1 idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time) - if connection.last_message_time > last_message_time and not connection.is_private_ip: - # Message from local IPs does not means internet connection - last_message_time = connection.last_message_time if connection.unpacker and idle > 30: # Delete the unpacker if not needed @@ -311,18 +314,6 @@ class ConnectionServer(object): # Reset bad action counter every 30 min connection.bad_actions = 0 - # Internet outage detection - if time.time() - last_message_time > max(60, 60 * 10 / max(1, float(len(self.connections)) / 50)): - # Offline: Last message more than 60-600sec depending on connection number - if self.has_internet and last_message_time: - self.has_internet = False - self.onInternetOffline() - else: - # Online - if not self.has_internet: - self.has_internet = True - self.onInternetOnline() - self.timecorrection = self.getTimecorrection() if time.time() - s > 0.01: @@ -353,6 +344,48 @@ class ConnectionServer(object): )) return num_closed + # Internet outage detection + def updateOnlineStatus(self, connection, outgoing_activity=False, successful_activity=False): + + now = time.time() + + if connection and not connection.is_private_ip: + if outgoing_activity: + self.last_outgoing_internet_activity_time = now + if successful_activity: + self.last_successful_internet_activity_time = now + self.setInternetStatus(True) + return + + if not self.last_outgoing_internet_activity_time: + return + + if ( + (self.last_successful_internet_activity_time < now - self.internet_outage_threshold) + and + (self.last_successful_internet_activity_time < self.last_outgoing_internet_activity_time) + ): + self.setInternetStatus(False) + return + + # This is the old algorithm just in case we missed something + idle = now - self.last_successful_internet_activity_time + if idle > max(60, 60 * 10 / max(1, float(len(self.connections)) / 50)): + # Offline: Last successful activity more than 60-600sec depending on connection number + self.setInternetStatus(False) + return + + def setInternetStatus(self, status): + if self.has_internet == status: + return + + self.has_internet = status + + if self.has_internet: + gevent.spawn(self.onInternetOnline) + else: + gevent.spawn(self.onInternetOffline) + def onInternetOnline(self): self.log.info("Internet online") diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 8294d179..c425b3f4 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -238,17 +238,50 @@ class FileServer(ConnectionServer): return res + # Returns False if Internet is immediately available + # Returns True if we've spent some time waiting for Internet + def waitForInternetOnline(self): + if self.has_internet: + return False + + while not self.has_internet: + time.sleep(15) + if self.has_internet: + break + if not self.check_pool.full(): + self.check_pool.spawn(self.updateRandomSite) + + return True + + def updateRandomSite(self, site_addresses=None, check_files=False): + if not site_addresses: + site_addresses = self.getSiteAddresses() + + site_addresses = random.sample(site_addresses, 1) + if len(site_addresses) < 1: + return + + address = site_addresses[0] + site = self.sites.get(address, None) + + if not site or not site.isServing(): + return + + log.debug("Checking randomly chosen site: %s", site.address_short) + + self.checkSite(site, check_files=check_files) + # Check site file integrity def checkSite(self, site, check_files=False): - if not site.isServing(): + if not site or not site.isServing(): return quick_start = len(site.peers) >= 50 if quick_start: - log.debug("Checking site: %s (quick start)" % (site.address)) + log.debug("Checking site: %s (quick start)", site.address_short) else: - log.debug("Checking site: %s" % (site.address)) + log.debug("Checking site: %s", site.address_short) if quick_start: site.setDelayedStartupAnnounce() @@ -291,7 +324,12 @@ class FileServer(ConnectionServer): if (not site) or (not site.isServing()): continue - check_thread = self.check_pool.spawn(self.checkSite, site, check_files) # Check in new thread + + while 1: + self.waitForInternetOnline() + self.check_pool.spawn(self.checkSite, site, check_files) + if not self.waitForInternetOnline(): + break if time.time() - progress_print_time > 60: progress_print_time = time.time() @@ -313,21 +351,25 @@ class FileServer(ConnectionServer): else: time.sleep(1) - log.info("Checking sites: finished in %.3fs" % (time.time() - start_time)) + log.info("Checking sites: finished in %.2fs" % (time.time() - start_time)) - def sitesMaintenanceThread(self): - import gc + def sitesMaintenanceThread(self, mode="full"): startup = True short_timeout = 2 - long_timeout = 60 * 2 + min_long_timeout = 10 + max_long_timeout = 60 * 10 + long_timeout = min_long_timeout + short_cycle_time_limit = 60 * 2 while 1: time.sleep(long_timeout) - gc.collect() # Explicit garbage collection + + start_time = time.time() log.debug( - "Starting maintenance cycle: connections=%s, internet=%s", + "Starting <%s> maintenance cycle: connections=%s, internet=%s", + mode, len(self.connections), self.has_internet ) start_time = time.time() @@ -341,7 +383,7 @@ class FileServer(ConnectionServer): if (not site) or (not site.isServing()): continue - log.debug("Running maintenance for site: %s", site.address) + log.debug("Running maintenance for site: %s", site.address_short) done = site.runPeriodicMaintenance(startup=startup) site = None @@ -349,15 +391,68 @@ class FileServer(ConnectionServer): sites_processed += 1 time.sleep(short_timeout) - log.debug("Maintenance cycle finished in %.3fs. Total sites: %d. Processed sites: %d", + # If we host hundreds of sites, the full maintenance cycle may take very + # long time, especially on startup ( > 1 hour). + # This means we are not able to run the maintenance procedure for active + # sites frequently enough using just a single maintenance thread. + # So we run 2 maintenance threads: + # * One running full cycles. + # * And one running short cycles for the most active sites. + # When the short cycle runs out of the time limit, it restarts + # from the beginning of the site list. + if mode == "short" and time.time() - start_time > short_cycle_time_limit: + break + + log.debug("<%s> maintenance cycle finished in %.2fs. Total sites: %d. Processed sites: %d. Timeout: %d", + mode, time.time() - start_time, len(site_addresses), - sites_processed + sites_processed, + long_timeout ) + if sites_processed: + long_timeout = max(int(long_timeout / 2), min_long_timeout) + else: + long_timeout = min(long_timeout + 1, max_long_timeout) + site_addresses = None startup = False + def keepAliveThread(self): + # This thread is mostly useless on a loaded system, since it never does + # any works, if we have active traffic. + # + # We should initiate some network activity to detect the Internet outage + # and avoid false positives. We normally have some network activity + # initiated by various parts on the application as well as network peers. + # So it's not a problem. + # + # However, if it actually happens that we have no network traffic for + # some time (say, we host just a couple of inactive sites, and no peers + # are interested in connecting to them), we initiate some traffic by + # performing the update for a random site. It's way better than just + # silly pinging a random peer for no profit. + while 1: + threshold = self.internet_outage_threshold / 2.0 + time.sleep(threshold / 2.0) + self.waitForInternetOnline() + last_activity_time = max( + self.last_successful_internet_activity_time, + self.last_outgoing_internet_activity_time) + now = time.time() + if not len(self.sites): + continue + if last_activity_time > now - threshold: + continue + if self.check_pool.full(): + continue + + log.info("No network activity for %.2fs. Running an update for a random site.", + now - last_activity_time + ) + self.check_pool.spawn(self.updateRandomSite) + # Periodic reloading of tracker files def reloadTrackerFilesThread(self): # TODO: @@ -420,7 +515,9 @@ class FileServer(ConnectionServer): gevent.spawn(self.checkSites) thread_reaload_tracker_files = gevent.spawn(self.reloadTrackerFilesThread) - thread_sites_maintenance = gevent.spawn(self.sitesMaintenanceThread) + thread_sites_maintenance_full = gevent.spawn(self.sitesMaintenanceThread, mode="full") + thread_sites_maintenance_short = gevent.spawn(self.sitesMaintenanceThread, mode="short") + thread_keep_alive = gevent.spawn(self.keepAliveThread) thread_wakeup_watcher = gevent.spawn(self.wakeupWatcher) ConnectionServer.listen(self) diff --git a/src/Site/Site.py b/src/Site/Site.py index b01e9aae..13ee730b 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -1177,6 +1177,7 @@ class Site(object): self.log.debug("Connected: %s, Need to close: %s, Closed: %s" % ( len(connected_peers), need_to_close, closed)) + @util.Noparallel(queue=True) def runPeriodicMaintenance(self, startup=False, force=False): if not self.isServing(): return False From e8358ee8f2df8e1b66b9a823a6ea6b28859300bc Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 31 Oct 2020 03:59:54 +0700 Subject: [PATCH 041/114] More fixes on the way to reliable site updates. --- src/Connection/ConnectionServer.py | 3 ++ src/File/FileServer.py | 70 +++++++++++------------------- src/Site/Site.py | 56 +++++++++++++++++++++--- 3 files changed, 79 insertions(+), 50 deletions(-) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 66b50608..2b4586e9 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -386,6 +386,9 @@ class ConnectionServer(object): else: gevent.spawn(self.onInternetOffline) + def isInternetOnline(self): + return self.has_internet + def onInternetOnline(self): self.log.info("Internet online") diff --git a/src/File/FileServer.py b/src/File/FileServer.py index c425b3f4..de14a9c9 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -33,8 +33,8 @@ class FileServer(ConnectionServer): # self.log = logging.getLogger("FileServer") # The value of self.log will be overwritten in ConnectionServer.__init__() - self.check_pool = gevent.pool.Pool(5) - self.check_start_time = 0 + self.update_pool = gevent.pool.Pool(5) + self.update_start_time = 0 self.supported_ip_types = ["ipv4"] # Outgoing ip_type support if helper.getIpType(ip) == "ipv6" or self.isIpv6Supported(): @@ -153,13 +153,12 @@ class FileServer(ConnectionServer): log.debug("FileRequest: %s %s" % (str(connection), message["cmd"])) req = FileRequest(self, connection) req.route(message["cmd"], message.get("req_id"), message.get("params")) - if not self.has_internet and not connection.is_private_ip: - self.has_internet = True - self.onInternetOnline() + if not connection.is_private_ip: + self.setInternetStatus(self, True) def onInternetOnline(self): log.info("Internet online") - gevent.spawn(self.checkSites, check_files=False, force_port_check=True) + gevent.spawn(self.updateSites, force_port_check=True) # Reload the FileRequest class to prevent restarts in debug mode def reload(self): @@ -241,19 +240,19 @@ class FileServer(ConnectionServer): # Returns False if Internet is immediately available # Returns True if we've spent some time waiting for Internet def waitForInternetOnline(self): - if self.has_internet: + if self.isInternetOnline(): return False - while not self.has_internet: + while not self.isInternetOnline(): time.sleep(15) - if self.has_internet: + if self.isInternetOnline(): break - if not self.check_pool.full(): - self.check_pool.spawn(self.updateRandomSite) + if not self.update_pool.full(): + self.update_pool.spawn(self.updateRandomSite) return True - def updateRandomSite(self, site_addresses=None, check_files=False): + def updateRandomSite(self, site_addresses=None): if not site_addresses: site_addresses = self.getSiteAddresses() @@ -269,39 +268,22 @@ class FileServer(ConnectionServer): log.debug("Checking randomly chosen site: %s", site.address_short) - self.checkSite(site, check_files=check_files) + self.updateSite(site) - # Check site file integrity - def checkSite(self, site, check_files=False): + def updateSite(self, site): if not site or not site.isServing(): return + site.considerUpdate() - quick_start = len(site.peers) >= 50 - - if quick_start: - log.debug("Checking site: %s (quick start)", site.address_short) - else: - log.debug("Checking site: %s", site.address_short) - - if quick_start: - site.setDelayedStartupAnnounce() - else: - site.announce(mode="startup") - - site.update(check_files=check_files) # Update site's content.json and download changed files - site.sendMyHashfield() - site.updateHashfield() - - # Check sites integrity @util.Noparallel() - def checkSites(self, check_files=False, force_port_check=False): - log.info("Checking sites: check_files=%s, force_port_check=%s", check_files, force_port_check) + def updateSites(self, force_port_check=False): + log.info("Checking sites: force_port_check=%s", force_port_check) # Don't wait port opening on first startup. Do the instant check now. if len(self.sites) <= 2: sites_checking = True for address, site in list(self.sites.items()): - gevent.spawn(self.checkSite, site, check_files) + gevent.spawn(self.updateSite, site) # Test and open port if not tested yet if not self.port_opened or force_port_check: @@ -313,7 +295,7 @@ class FileServer(ConnectionServer): sites_processed = 0 start_time = time.time() - self.check_start_time = start_time + self.update_start_time = start_time progress_print_time = time.time() # Check sites integrity @@ -327,7 +309,7 @@ class FileServer(ConnectionServer): while 1: self.waitForInternetOnline() - self.check_pool.spawn(self.checkSite, site, check_files) + self.update_pool.spawn(self.updateSite, site) if not self.waitForInternetOnline(): break @@ -345,8 +327,8 @@ class FileServer(ConnectionServer): time_left ) - if (self.check_start_time != start_time) and self.check_pool.full(): - # Another check is running, throttling... + if (self.update_start_time != start_time) and self.update_pool.full(): + # Another updateSites() is running, throttling now... time.sleep(5) else: time.sleep(1) @@ -370,7 +352,7 @@ class FileServer(ConnectionServer): log.debug( "Starting <%s> maintenance cycle: connections=%s, internet=%s", mode, - len(self.connections), self.has_internet + len(self.connections), self.isInternetOnline() ) start_time = time.time() @@ -445,13 +427,13 @@ class FileServer(ConnectionServer): continue if last_activity_time > now - threshold: continue - if self.check_pool.full(): + if self.update_pool.full(): continue log.info("No network activity for %.2fs. Running an update for a random site.", now - last_activity_time ) - self.check_pool.spawn(self.updateRandomSite) + self.update_pool.spawn(self.updateRandomSite) # Periodic reloading of tracker files def reloadTrackerFilesThread(self): @@ -484,7 +466,7 @@ class FileServer(ConnectionServer): log.info("IP change detected from %s to %s" % (last_my_ips, my_ips)) if is_time_changed or is_ip_changed: - self.checkSites(check_files=False, force_port_check=True) + self.updateSites(force_port_check=True) last_time = time.time() last_my_ips = my_ips @@ -512,7 +494,7 @@ class FileServer(ConnectionServer): DebugReloader.watcher.addCallback(self.reload) if check_sites: # Open port, Update sites, Check files integrity - gevent.spawn(self.checkSites) + gevent.spawn(self.updateSites) thread_reaload_tracker_files = gevent.spawn(self.reloadTrackerFilesThread) thread_sites_maintenance_full = gevent.spawn(self.sitesMaintenanceThread, mode="full") diff --git a/src/Site/Site.py b/src/Site/Site.py index 13ee730b..6d72e119 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -76,7 +76,7 @@ class Site(object): ScaledTimeoutHandler(60 * 30, 60 * 2, handler=self.periodicMaintenanceHandler_announce, scaler=self.getAnnounceRating), - ScaledTimeoutHandler(60 * 20, 60 * 10, + ScaledTimeoutHandler(60 * 30, 60 * 5, handler=self.periodicMaintenanceHandler_general, scaler=self.getActivityRating) ] @@ -91,6 +91,8 @@ class Site(object): self.worker_manager = WorkerManager(self) # Handle site download from other peers self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept) self.content_updated = None # Content.js update time + self.last_check_files_time = 0 + self.last_online_update = 0 self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout] self.page_requested = False # Page viewed in browser self.websockets = [] # Active site websocket connections @@ -564,6 +566,44 @@ class Site(object): self.updateWebsocket(updated=True) + def considerUpdate(self): + if not self.isServing(): + return + + online = self.connection_server.isInternetOnline() + + if online and time.time() - self.last_online_update < 60 * 10: + with gevent.Timeout(10, exception=False): + self.announcer.announcePex() + return + + # TODO: there should be a configuration options controlling: + # * whether to check files on the program startup + # * whether to check files during the run time and how often + check_files = self.last_check_files_time == 0 + + self.last_check_files_time = time.time() + + # quick start, avoiding redundant announces + if len(self.peers) >= 50: + if len(self.getConnectedPeers()) > 4: + pass # Don't run announce() at all + else: + self.setDelayedStartupAnnounce() + else: + self.announce(mode="startup") + + online = online and self.connection_server.isInternetOnline() + + self.update(check_files=check_files) + self.sendMyHashfield() + self.updateHashfield() + + online = online and self.connection_server.isInternetOnline() + + if online: + self.last_online_update = time.time() + # Update site by redownload all content.json def redownloadContents(self): # Download all content.json again @@ -927,6 +967,14 @@ class Site(object): if (not force_safe) and self.settings["own"]: rating = min(rating, 0.6) + if self.content_updated is False: # Last check modification failed + rating += 0.1 + elif self.bad_files: + rating += 0.1 + + if rating > 1.0: + rating = 1.0 + return rating def getAnnounceRating(self): @@ -1210,14 +1258,10 @@ class Site(object): with gevent.Timeout(10, exception=False): self.announcer.announcePex() + self.update() self.sendMyHashfield(3) self.updateHashfield(3) - if self.content_updated is False: # Last check modification failed - self.update() - elif self.bad_files: - self.retryBadFiles() - return True def periodicMaintenanceHandler_announce(self, startup=False, force=False): From ea21b32b93384cc718aada5fd184de5dbbf23b1f Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 31 Oct 2020 18:05:50 +0700 Subject: [PATCH 042/114] Add explicit invalidation and expiration of site update timestamps --- src/Connection/ConnectionServer.py | 4 ++ src/File/FileServer.py | 109 +++++++++++++++++++++-------- src/Site/Site.py | 101 +++++++++++++++----------- 3 files changed, 142 insertions(+), 72 deletions(-) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 2b4586e9..3ec0932d 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -43,6 +43,8 @@ class ConnectionServer(object): self.ips = {} # Connection by ip self.has_internet = True # Internet outage detection + self.internet_online_since = 0 + self.internet_offline_since = 0 self.last_outgoing_internet_activity_time = 0 # Last time the application tried to send any data self.last_successful_internet_activity_time = 0 # Last time the application successfully sent or received any data self.internet_outage_threshold = 60 * 2 @@ -382,8 +384,10 @@ class ConnectionServer(object): self.has_internet = status if self.has_internet: + self.internet_online_since = time.time() gevent.spawn(self.onInternetOnline) else: + self.internet_offline_since = time.time() gevent.spawn(self.onInternetOffline) def isInternetOnline(self): diff --git a/src/File/FileServer.py b/src/File/FileServer.py index de14a9c9..66c8c135 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -33,8 +33,11 @@ class FileServer(ConnectionServer): # self.log = logging.getLogger("FileServer") # The value of self.log will be overwritten in ConnectionServer.__init__() + self.recheck_port = True + self.update_pool = gevent.pool.Pool(5) self.update_start_time = 0 + self.update_sites_task_next_nr = 1 self.supported_ip_types = ["ipv4"] # Outgoing ip_type support if helper.getIpType(ip) == "ipv6" or self.isIpv6Supported(): @@ -158,7 +161,13 @@ class FileServer(ConnectionServer): def onInternetOnline(self): log.info("Internet online") - gevent.spawn(self.updateSites, force_port_check=True) + invalid_interval=( + self.internet_offline_since - self.internet_outage_threshold - random.randint(60 * 5, 60 * 10), + time.time() + ) + self.invalidateUpdateTime(invalid_interval) + self.recheck_port = True + gevent.spawn(self.updateSites) # Reload the FileRequest class to prevent restarts in debug mode def reload(self): @@ -237,6 +246,17 @@ class FileServer(ConnectionServer): return res + @util.Noparallel(queue=True) + def recheckPort(self): + if not self.recheck_port: + return + + if not self.port_opened or self.recheck_port: + self.portCheck() + if not self.port_opened["ipv4"]: + self.tor_manager.startOnions() + self.recheck_port = False + # Returns False if Internet is immediately available # Returns True if we've spent some time waiting for Internet def waitForInternetOnline(self): @@ -250,6 +270,7 @@ class FileServer(ConnectionServer): if not self.update_pool.full(): self.update_pool.spawn(self.updateRandomSite) + self.recheckPort() return True def updateRandomSite(self, site_addresses=None): @@ -270,46 +291,68 @@ class FileServer(ConnectionServer): self.updateSite(site) - def updateSite(self, site): + def updateSite(self, site, check_files=False, dry_run=False): if not site or not site.isServing(): - return - site.considerUpdate() + return False + return site.considerUpdate(check_files=check_files, dry_run=dry_run) - @util.Noparallel() - def updateSites(self, force_port_check=False): - log.info("Checking sites: force_port_check=%s", force_port_check) + def getSite(self, address): + return self.sites.get(address, None) + + def invalidateUpdateTime(self, invalid_interval): + for address in self.getSiteAddresses(): + site = self.getSite(address) + if site: + site.invalidateUpdateTime(invalid_interval) + + def updateSites(self, check_files=False): + task_nr = self.update_sites_task_next_nr + self.update_sites_task_next_nr += 1 + + task_description = "updateSites: #%d, check_files=%s" % (task_nr, check_files) + log.info("%s: started", task_description) # Don't wait port opening on first startup. Do the instant check now. if len(self.sites) <= 2: - sites_checking = True for address, site in list(self.sites.items()): - gevent.spawn(self.updateSite, site) + self.updateSite(site, check_files=check_files) - # Test and open port if not tested yet - if not self.port_opened or force_port_check: - self.portCheck() - if not self.port_opened["ipv4"]: - self.tor_manager.startOnions() + all_site_addresses = self.getSiteAddresses() + site_addresses = [ + address for address in all_site_addresses + if self.updateSite(self.getSite(address), check_files=check_files, dry_run=True) + ] - site_addresses = self.getSiteAddresses() + log.info("%s: chosen %d sites (of %d)", task_description, len(site_addresses), len(all_site_addresses)) sites_processed = 0 + sites_skipped = 0 start_time = time.time() self.update_start_time = start_time progress_print_time = time.time() # Check sites integrity for site_address in site_addresses: - site = self.sites.get(site_address, None) + if check_files: + time.sleep(10) + else: + time.sleep(1) + + site = self.getSite(site_address) + if not self.updateSite(site, check_files=check_files, dry_run=True): + sites_skipped += 1 + continue sites_processed += 1 - if (not site) or (not site.isServing()): - continue - while 1: self.waitForInternetOnline() - self.update_pool.spawn(self.updateSite, site) + thread = self.update_pool.spawn(self.updateSite, + site, check_files=check_files) + if check_files: + # Limit the concurency + # ZeroNet may be laggy when running from HDD. + thread.join(timeout=60) if not self.waitForInternetOnline(): break @@ -319,21 +362,17 @@ class FileServer(ConnectionServer): time_per_site = time_spent / float(sites_processed) sites_left = len(site_addresses) - sites_processed time_left = time_per_site * sites_left - log.info("Checking sites: DONE: %d sites in %.2fs (%.2fs per site); LEFT: %d sites in %.2fs", + log.info("%s: DONE: %d sites in %.2fs (%.2fs per site); SKIPPED: %d sites; LEFT: %d sites in %.2fs", + task_description, sites_processed, time_spent, time_per_site, + sites_skipped, sites_left, time_left ) - if (self.update_start_time != start_time) and self.update_pool.full(): - # Another updateSites() is running, throttling now... - time.sleep(5) - else: - time.sleep(1) - - log.info("Checking sites: finished in %.2fs" % (time.time() - start_time)) + log.info("%s: finished in %.2fs", task_description, time.time() - start_time) def sitesMaintenanceThread(self, mode="full"): startup = True @@ -402,7 +441,7 @@ class FileServer(ConnectionServer): startup = False def keepAliveThread(self): - # This thread is mostly useless on a loaded system, since it never does + # This thread is mostly useless on a system under load, since it never does # any works, if we have active traffic. # # We should initiate some network activity to detect the Internet outage @@ -466,7 +505,13 @@ class FileServer(ConnectionServer): log.info("IP change detected from %s to %s" % (last_my_ips, my_ips)) if is_time_changed or is_ip_changed: - self.updateSites(force_port_check=True) + invalid_interval=( + last_time - self.internet_outage_threshold - random.randint(60 * 5, 60 * 10), + time.time() + ) + self.invalidateUpdateTime(invalid_interval) + self.recheck_port = True + gevent.spawn(self.updateSites) last_time = time.time() last_my_ips = my_ips @@ -494,7 +539,9 @@ class FileServer(ConnectionServer): DebugReloader.watcher.addCallback(self.reload) if check_sites: # Open port, Update sites, Check files integrity - gevent.spawn(self.updateSites) + gevent.spawn(self.updateSites, check_files=True) + + gevent.spawn(self.updateSites) thread_reaload_tracker_files = gevent.spawn(self.reloadTrackerFilesThread) thread_sites_maintenance_full = gevent.spawn(self.sitesMaintenanceThread, mode="full") diff --git a/src/Site/Site.py b/src/Site/Site.py index 6d72e119..14077aae 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -81,8 +81,6 @@ class Site(object): scaler=self.getActivityRating) ] - self.delayed_startup_announce = False - self.content = None # Load content.json self.peers = {} # Key: ip:port, Value: Peer.Peer self.peers_recent = collections.deque(maxlen=150) @@ -93,6 +91,7 @@ class Site(object): self.content_updated = None # Content.js update time self.last_check_files_time = 0 self.last_online_update = 0 + self.startup_announce_done = 0 self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout] self.page_requested = False # Page viewed in browser self.websockets = [] # Active site websocket connections @@ -525,6 +524,28 @@ class Site(object): time.sleep(0.1) return queried + def invalidateUpdateTime(self, invalid_interval): + a, b = invalid_interval + if b is None: + b = time.time() + if a is None: + a = b + if a <= self.last_online_update and self.last_online_update <= b: + self.last_online_update = 0 + self.log.info("Update time invalidated") + + def isUpdateTimeValid(self): + if not self.last_online_update: + return False + expirationThreshold = 60 * 60 * 6 + return self.last_online_update > time.time() - expirationThreshold + + def refreshUpdateTime(self, valid=True): + if valid: + self.last_online_update = time.time() + else: + self.last_online_update = 0 + # Update content.json from peers and download changed files # Return: None @util.Noparallel() @@ -564,45 +585,56 @@ class Site(object): else: self.content_updated = time.time() + self.sendMyHashfield() + self.updateHashfield() + + self.refreshUpdateTime(valid=self.connection_server.isInternetOnline()) + self.updateWebsocket(updated=True) - def considerUpdate(self): + @util.Noparallel(queue=True, ignore_args=True) + def considerUpdate(self, check_files=False, dry_run=False): if not self.isServing(): - return + return False online = self.connection_server.isInternetOnline() - if online and time.time() - self.last_online_update < 60 * 10: - with gevent.Timeout(10, exception=False): - self.announcer.announcePex() - return + run_update = False + msg = None + + if not online: + run_update = True + msg = "network connection seems broken, trying to update the site to check if the network is up" + elif check_files: + run_update = True + msg = "checking site's files..." + elif not self.isUpdateTimeValid(): + run_update = True + msg = "update time is not invalid, updating now..." + + if not run_update: + return False + + if dry_run: + return True + + self.log.debug(msg) # TODO: there should be a configuration options controlling: # * whether to check files on the program startup # * whether to check files during the run time and how often - check_files = self.last_check_files_time == 0 - + check_files = check_files and (self.last_check_files_time == 0) self.last_check_files_time = time.time() - # quick start, avoiding redundant announces - if len(self.peers) >= 50: - if len(self.getConnectedPeers()) > 4: - pass # Don't run announce() at all - else: - self.setDelayedStartupAnnounce() - else: - self.announce(mode="startup") - - online = online and self.connection_server.isInternetOnline() + if len(self.peers) < 50: + self.announce(mode="update") self.update(check_files=check_files) - self.sendMyHashfield() - self.updateHashfield() online = online and self.connection_server.isInternetOnline() + self.refreshUpdateTime(valid=online) - if online: - self.last_online_update = time.time() + return True # Update site by redownload all content.json def redownloadContents(self): @@ -932,16 +964,6 @@ class Site(object): peer.found(source) return peer - def setDelayedStartupAnnounce(self): - self.delayed_startup_announce = True - - def applyDelayedStartupAnnounce(self): - if self.delayed_startup_announce: - self.delayed_startup_announce = False - self.announce(mode="startup") - return True - return False - def announce(self, *args, **kwargs): if self.isServing(): self.announcer.announce(*args, **kwargs) @@ -1243,8 +1265,6 @@ class Site(object): if not self.isServing(): return False - self.applyDelayedStartupAnnounce() - if not self.peers: return False @@ -1259,8 +1279,6 @@ class Site(object): self.announcer.announcePex() self.update() - self.sendMyHashfield(3) - self.updateHashfield(3) return True @@ -1270,10 +1288,11 @@ class Site(object): self.log.debug("periodicMaintenanceHandler_announce: startup=%s, force=%s" % (startup, force)) - if self.applyDelayedStartupAnnounce(): - return True + if startup and len(self.peers) < 10: + self.announce(mode="startup") + else: + self.announce(mode="update", pex=False) - self.announce(mode="update", pex=False) return True # Send hashfield to peers From ba16fdcae9473025ce5bf930ed8733ee476a6b3e Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 2 Nov 2020 09:04:38 +0700 Subject: [PATCH 043/114] Fix a typo --- src/Site/Site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index 14077aae..e808b706 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -610,7 +610,7 @@ class Site(object): msg = "checking site's files..." elif not self.isUpdateTimeValid(): run_update = True - msg = "update time is not invalid, updating now..." + msg = "update time is not valid, updating now..." if not run_update: return False From c84b413f58bfc810f1e56b7e6414367f81efadf6 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 3 Nov 2020 21:21:33 +0700 Subject: [PATCH 044/114] Refactor ConnectionServer, FileServer; fix bugs introduced in previous commits --- src/Connection/ConnectionServer.py | 89 +++++++++++++++-- src/File/FileServer.py | 148 ++++++++++++++++++----------- src/Site/Site.py | 66 ++++++++----- src/Site/SiteAnnouncer.py | 6 ++ 4 files changed, 222 insertions(+), 87 deletions(-) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 3ec0932d..b66d1739 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -8,6 +8,7 @@ import gevent import msgpack from gevent.server import StreamServer from gevent.pool import Pool +import gevent.event import util from util import helper @@ -35,6 +36,8 @@ class ConnectionServer(object): self.port_opened = {} self.peer_blacklist = SiteManager.peer_blacklist + self.managed_pools = {} + self.tor_manager = TorManager(self.ip, self.port) self.connections = [] # Connections self.whitelist = config.ip_local # No flood protection on this ips @@ -53,8 +56,12 @@ class ConnectionServer(object): self.stream_server_proxy = None self.running = False self.stopping = False + self.stopping_event = gevent.event.Event() self.thread_checker = None + self.thread_pool = Pool(None) + self.managed_pools["thread"] = self.thread_pool + self.stat_recv = defaultdict(lambda: defaultdict(int)) self.stat_sent = defaultdict(lambda: defaultdict(int)) self.bytes_recv = 0 @@ -70,6 +77,7 @@ class ConnectionServer(object): self.timecorrection = 0.0 self.pool = Pool(500) # do not accept more than 500 connections + self.managed_pools["incoming"] = self.pool # Bittorrent style peerid self.peer_id = "-UT3530-%s" % CryptHash.random(12, "base64") @@ -90,7 +98,7 @@ class ConnectionServer(object): return False self.running = True if check_connections: - self.thread_checker = gevent.spawn(self.checkConnections) + self.thread_checker = self.spawn(self.checkConnections) CryptConnection.manager.loadCerts() if config.tor != "disable": self.tor_manager.start() @@ -114,7 +122,7 @@ class ConnectionServer(object): return None if self.stream_server_proxy: - gevent.spawn(self.listenProxy) + self.spawn(self.listenProxy) try: self.stream_server.serve_forever() except Exception as err: @@ -126,18 +134,65 @@ class ConnectionServer(object): self.log.debug("Stopping %s" % self.stream_server) self.stopping = True self.running = False + self.stopping_event.set() + self.onStop() + + def onStop(self): + prev_sizes = {} + for i in range(60): + sizes = {} + total_size = 0 + + for name, pool in self.managed_pools.items(): + pool.join(timeout=1) + size = len(pool) + sizes[name] = size + total_size += size + + if total_size == 0: + break + + if prev_sizes != sizes: + s = "" + for name, size in sizes.items(): + s += "%s pool: %s, " % (name, size) + s += "total: %s" % total_size + + self.log.info("Waiting for tasks in managed pools to stop: %s", s) + + prev_sizes = sizes + + for name, pool in self.managed_pools.items(): + size = len(pool) + if size: + self.log.info("Killing %s tasks in %s pool", size, name) + pool.kill() + if self.thread_checker: gevent.kill(self.thread_checker) + self.thread_checker = None if self.stream_server: self.stream_server.stop() + # Sleeps the specified amount of time or until ConnectionServer is stopped + def sleep(self, t): + if t: + self.stopping_event.wait(timeout=t) + else: + time.sleep(t) + + # Spawns a thread that will be waited for on server being stooped (and killed after a timeout) + def spawn(self, *args, **kwargs): + thread = self.thread_pool.spawn(*args, **kwargs) + return thread + def closeConnections(self): self.log.debug("Closing all connection: %s" % len(self.connections)) for connection in self.connections[:]: connection.close("Close all connections") def handleIncomingConnection(self, sock, addr): - if config.offline: + if self.allowsAcceptingConnections(): sock.close() return False @@ -155,7 +210,7 @@ class ConnectionServer(object): self.ip_incoming[ip] += 1 if self.ip_incoming[ip] > 6: # Allow 6 in 1 minute from same ip self.log.debug("Connection flood detected from %s" % ip) - time.sleep(30) + self.sleep(30) sock.close() return False else: @@ -207,7 +262,7 @@ class ConnectionServer(object): return connection # No connection found - if create and not config.offline: # Allow to create new connection if not found + if create and self.allowsCreatingConnections(): if port == 0: raise Exception("This peer is not connectable") @@ -233,7 +288,7 @@ class ConnectionServer(object): raise err if len(self.connections) > config.global_connected_limit: - gevent.spawn(self.checkMaxConnections) + self.spawn(self.checkMaxConnections) return connection else: @@ -256,7 +311,7 @@ class ConnectionServer(object): def checkConnections(self): run_i = 0 - time.sleep(15) + self.sleep(15) while self.running: run_i += 1 self.ip_incoming = {} # Reset connected ips counter @@ -321,7 +376,7 @@ class ConnectionServer(object): if time.time() - s > 0.01: self.log.debug("Connection cleanup in %.3fs" % (time.time() - s)) - time.sleep(15) + self.sleep(15) self.log.debug("Checkconnections ended") @util.Noparallel(blocking=False) @@ -385,10 +440,10 @@ class ConnectionServer(object): if self.has_internet: self.internet_online_since = time.time() - gevent.spawn(self.onInternetOnline) + self.spawn(self.onInternetOnline) else: self.internet_offline_since = time.time() - gevent.spawn(self.onInternetOffline) + self.spawn(self.onInternetOffline) def isInternetOnline(self): return self.has_internet @@ -400,6 +455,20 @@ class ConnectionServer(object): self.had_external_incoming = False self.log.info("Internet offline") + def allowsCreatingConnections(self): + if config.offline: + return False + if self.stopping: + return False + return True + + def allowsAcceptingConnections(self): + if config.offline: + return False + if self.stopping: + return False + return True + def getTimecorrection(self): corrections = sorted([ connection.handshake.get("time") - connection.handshake_time + connection.last_ping_delay diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 66c8c135..b228680a 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -64,6 +64,8 @@ class FileServer(ConnectionServer): ConnectionServer.__init__(self, ip, port, self.handleRequest) log.debug("Supported IP types: %s" % self.supported_ip_types) + self.managed_pools["update"] = self.pool + if ip_type == "dual" and ip == "::": # Also bind to ipv4 addres in dual mode try: @@ -76,18 +78,28 @@ class FileServer(ConnectionServer): self.port_opened = {} - self.sites = self.site_manager.sites self.last_request = time.time() self.files_parsing = {} self.ui_server = None + def getSites(self): + sites = self.site_manager.list() + # We need to keep self.sites for the backward compatibility with plugins. + # Never. Ever. Use it. + # TODO: fix plugins + self.sites = sites + return sites + + def getSite(self, address): + return self.getSites().get(address, None) + def getSiteAddresses(self): # Avoid saving the site list on the stack, since a site may be deleted # from the original list while iterating. # Use the list of addresses instead. return [ site.address for site in - sorted(list(self.sites.values()), key=lambda site: site.settings.get("modified", 0), reverse=True) + sorted(list(self.getSites().values()), key=lambda site: site.settings.get("modified", 0), reverse=True) ] def getRandomPort(self, ip, port_range_from, port_range_to): @@ -110,7 +122,7 @@ class FileServer(ConnectionServer): log.info("Found unused random port: %s" % port) return port else: - time.sleep(0.1) + self.sleep(0.1) return False def isIpv6Supported(self): @@ -157,7 +169,7 @@ class FileServer(ConnectionServer): req = FileRequest(self, connection) req.route(message["cmd"], message.get("req_id"), message.get("params")) if not connection.is_private_ip: - self.setInternetStatus(self, True) + self.setInternetStatus(True) def onInternetOnline(self): log.info("Internet online") @@ -167,7 +179,7 @@ class FileServer(ConnectionServer): ) self.invalidateUpdateTime(invalid_interval) self.recheck_port = True - gevent.spawn(self.updateSites) + self.spawn(self.updateSites) # Reload the FileRequest class to prevent restarts in debug mode def reload(self): @@ -201,7 +213,7 @@ class FileServer(ConnectionServer): self.ui_server.updateWebsocket() if "ipv6" in self.supported_ip_types: - res_ipv6_thread = gevent.spawn(self.portchecker.portCheck, self.port, "ipv6") + res_ipv6_thread = self.spawn(self.portchecker.portCheck, self.port, "ipv6") else: res_ipv6_thread = None @@ -251,7 +263,7 @@ class FileServer(ConnectionServer): if not self.recheck_port: return - if not self.port_opened or self.recheck_port: + if not self.port_opened: self.portCheck() if not self.port_opened["ipv4"]: self.tor_manager.startOnions() @@ -259,21 +271,28 @@ class FileServer(ConnectionServer): # Returns False if Internet is immediately available # Returns True if we've spent some time waiting for Internet + # Returns None if FileServer is stopping or the Offline mode is enabled def waitForInternetOnline(self): + if config.offline or self.stopping: + return None + if self.isInternetOnline(): return False while not self.isInternetOnline(): - time.sleep(15) + self.sleep(30) + if config.offline or self.stopping: + return None if self.isInternetOnline(): break - if not self.update_pool.full(): - self.update_pool.spawn(self.updateRandomSite) + if len(self.update_pool) == 0: + thread = self.update_pool.spawn(self.updateRandomSite) + thread.join() self.recheckPort() return True - def updateRandomSite(self, site_addresses=None): + def updateRandomSite(self, site_addresses=None, force=False): if not site_addresses: site_addresses = self.getSiteAddresses() @@ -282,22 +301,19 @@ class FileServer(ConnectionServer): return address = site_addresses[0] - site = self.sites.get(address, None) + site = self.getSite(address) - if not site or not site.isServing(): + if not site: return log.debug("Checking randomly chosen site: %s", site.address_short) - self.updateSite(site) + self.updateSite(site, force=force) - def updateSite(self, site, check_files=False, dry_run=False): - if not site or not site.isServing(): + def updateSite(self, site, check_files=False, force=False, dry_run=False): + if not site: return False - return site.considerUpdate(check_files=check_files, dry_run=dry_run) - - def getSite(self, address): - return self.sites.get(address, None) + return site.considerUpdate(check_files=check_files, force=force, dry_run=dry_run) def invalidateUpdateTime(self, invalid_interval): for address in self.getSiteAddresses(): @@ -313,8 +329,8 @@ class FileServer(ConnectionServer): log.info("%s: started", task_description) # Don't wait port opening on first startup. Do the instant check now. - if len(self.sites) <= 2: - for address, site in list(self.sites.items()): + if len(self.getSites()) <= 2: + for address, site in list(self.getSites().items()): self.updateSite(site, check_files=check_files) all_site_addresses = self.getSiteAddresses() @@ -334,9 +350,12 @@ class FileServer(ConnectionServer): # Check sites integrity for site_address in site_addresses: if check_files: - time.sleep(10) + self.sleep(10) else: - time.sleep(1) + self.sleep(1) + + if self.stopping: + break site = self.getSite(site_address) if not self.updateSite(site, check_files=check_files, dry_run=True): @@ -345,10 +364,9 @@ class FileServer(ConnectionServer): sites_processed += 1 - while 1: + while self.running: self.waitForInternetOnline() - thread = self.update_pool.spawn(self.updateSite, - site, check_files=check_files) + thread = self.update_pool.spawn(self.updateSite, site, check_files=check_files) if check_files: # Limit the concurency # ZeroNet may be laggy when running from HDD. @@ -383,8 +401,11 @@ class FileServer(ConnectionServer): long_timeout = min_long_timeout short_cycle_time_limit = 60 * 2 - while 1: - time.sleep(long_timeout) + while self.running: + self.sleep(long_timeout) + + if self.stopping: + break start_time = time.time() @@ -400,8 +421,11 @@ class FileServer(ConnectionServer): sites_processed = 0 for site_address in site_addresses: - site = self.sites.get(site_address, None) - if (not site) or (not site.isServing()): + if self.stopping: + break + + site = self.getSite(site_address) + if not site: continue log.debug("Running maintenance for site: %s", site.address_short) @@ -410,7 +434,7 @@ class FileServer(ConnectionServer): site = None if done: sites_processed += 1 - time.sleep(short_timeout) + self.sleep(short_timeout) # If we host hundreds of sites, the full maintenance cycle may take very # long time, especially on startup ( > 1 hour). @@ -454,19 +478,24 @@ class FileServer(ConnectionServer): # are interested in connecting to them), we initiate some traffic by # performing the update for a random site. It's way better than just # silly pinging a random peer for no profit. - while 1: - threshold = self.internet_outage_threshold / 2.0 - time.sleep(threshold / 2.0) + while self.running: self.waitForInternetOnline() + + threshold = self.internet_outage_threshold / 2.0 + + self.sleep(threshold / 2.0) + if self.stopping: + break + last_activity_time = max( self.last_successful_internet_activity_time, self.last_outgoing_internet_activity_time) now = time.time() - if not len(self.sites): + if not len(self.getSites()): continue if last_activity_time > now - threshold: continue - if self.update_pool.full(): + if len(self.update_pool) == 0: continue log.info("No network activity for %.2fs. Running an update for a random site.", @@ -481,16 +510,18 @@ class FileServer(ConnectionServer): # We should check if the files have actually changed, # and do it more often. interval = 60 * 10 - while 1: - time.sleep(interval) + while self.running: + self.sleep(interval) + if self.stopping: + break config.loadTrackersFile() # Detects if computer back from wakeup def wakeupWatcher(self): last_time = time.time() last_my_ips = socket.gethostbyname_ex('')[2] - while 1: - time.sleep(30) + while self.running: + self.sleep(30) is_time_changed = time.time() - max(self.last_request, last_time) > 60 * 3 if is_time_changed: # If taken more than 3 minute then the computer was in sleep mode @@ -511,17 +542,28 @@ class FileServer(ConnectionServer): ) self.invalidateUpdateTime(invalid_interval) self.recheck_port = True - gevent.spawn(self.updateSites) + self.spawn(self.updateSites) last_time = time.time() last_my_ips = my_ips # Bind and start serving sites - def start(self, check_sites=True): + # If passive_mode is False, FileServer starts the full-featured file serving: + # * Checks for updates at startup. + # * Checks site's integrity. + # * Runs periodic update checks. + # * Watches for internet being up or down and for computer to wake up and runs update checks. + # If passive_mode is True, all the mentioned activity is disabled. + def start(self, passive_mode=False, check_sites=None, check_connections=True): + + # Backward compatibility for a misnamed argument: + if check_sites is not None: + passive_mode = not check_sites + if self.stopping: return False - ConnectionServer.start(self) + ConnectionServer.start(self, check_connections=check_connections) try: self.stream_server.start() @@ -532,22 +574,22 @@ class FileServer(ConnectionServer): sys.modules["main"].ui_server.stop() return False - self.sites = self.site_manager.list() if config.debug: # Auto reload FileRequest on change from Debug import DebugReloader DebugReloader.watcher.addCallback(self.reload) - if check_sites: # Open port, Update sites, Check files integrity - gevent.spawn(self.updateSites, check_files=True) + if not passive_mode: + self.spawn(self.updateSites) + thread_reaload_tracker_files = self.spawn(self.reloadTrackerFilesThread) + thread_sites_maintenance_full = self.spawn(self.sitesMaintenanceThread, mode="full") + thread_sites_maintenance_short = self.spawn(self.sitesMaintenanceThread, mode="short") + thread_keep_alive = self.spawn(self.keepAliveThread) + thread_wakeup_watcher = self.spawn(self.wakeupWatcher) - gevent.spawn(self.updateSites) + self.sleep(0.1) + self.spawn(self.updateSites, check_files=True) - thread_reaload_tracker_files = gevent.spawn(self.reloadTrackerFilesThread) - thread_sites_maintenance_full = gevent.spawn(self.sitesMaintenanceThread, mode="full") - thread_sites_maintenance_short = gevent.spawn(self.sitesMaintenanceThread, mode="short") - thread_keep_alive = gevent.spawn(self.keepAliveThread) - thread_wakeup_watcher = gevent.spawn(self.wakeupWatcher) ConnectionServer.listen(self) diff --git a/src/Site/Site.py b/src/Site/Site.py index e808b706..1581a106 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -172,6 +172,8 @@ class Site(object): def isServing(self): if config.offline: return False + elif self.connection_server.stopping: + return False else: return self.settings["serving"] @@ -532,7 +534,7 @@ class Site(object): a = b if a <= self.last_online_update and self.last_online_update <= b: self.last_online_update = 0 - self.log.info("Update time invalidated") + self.log.debug("Update time invalidated") def isUpdateTimeValid(self): if not self.last_online_update: @@ -593,33 +595,12 @@ class Site(object): self.updateWebsocket(updated=True) @util.Noparallel(queue=True, ignore_args=True) - def considerUpdate(self, check_files=False, dry_run=False): - if not self.isServing(): + def _considerUpdate_realJob(self, check_files=False, force=False): + if not self._considerUpdate_check(check_files=check_files, force=force, log_reason=True): return False online = self.connection_server.isInternetOnline() - run_update = False - msg = None - - if not online: - run_update = True - msg = "network connection seems broken, trying to update the site to check if the network is up" - elif check_files: - run_update = True - msg = "checking site's files..." - elif not self.isUpdateTimeValid(): - run_update = True - msg = "update time is not valid, updating now..." - - if not run_update: - return False - - if dry_run: - return True - - self.log.debug(msg) - # TODO: there should be a configuration options controlling: # * whether to check files on the program startup # * whether to check files during the run time and how often @@ -628,6 +609,7 @@ class Site(object): if len(self.peers) < 50: self.announce(mode="update") + online = online and self.connection_server.isInternetOnline() self.update(check_files=check_files) @@ -636,6 +618,39 @@ class Site(object): return True + def _considerUpdate_check(self, check_files=False, force=False, log_reason=False): + if not self.isServing(): + return False + + online = self.connection_server.isInternetOnline() + + run_update = False + msg = None + + if force: + run_update = True + msg = "forcing site update" + elif not online: + run_update = True + msg = "network connection seems broken, trying to update a site to check if the network is up" + elif check_files: + run_update = True + msg = "checking site's files..." + elif not self.isUpdateTimeValid(): + run_update = True + msg = "update time is not valid, updating now..." + + if run_update and log_reason: + self.log.debug(msg) + + return run_update + + def considerUpdate(self, check_files=False, force=False, dry_run=False): + run_update = self._considerUpdate_check(check_files=check_files, force=force) + if run_update and not dry_run: + run_update = self._considerUpdate_realJob(check_files=check_files, force=force) + return run_update + # Update site by redownload all content.json def redownloadContents(self): # Download all content.json again @@ -1085,6 +1100,9 @@ class Site(object): # Keep connections def needConnections(self, num=None, check_site_on_reconnect=False, pex=True): + if not self.connection_server.allowsCreatingConnections(): + return + if num is None: num = self.getPreferableActiveConnectionCount() diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index 5a97807e..6a510583 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -127,6 +127,9 @@ class SiteAnnouncer(object): @util.Noparallel() def announce(self, force=False, mode="start", pex=True): + if not self.site.isServing(): + return + if time.time() - self.time_last_announce < 30 and not force: return # No reannouncing within 30 secs @@ -300,6 +303,9 @@ class SiteAnnouncer(object): @util.Noparallel(blocking=False) def announcePex(self, query_num=2, need_num=10): + if not self.site.isServing(): + return + self.updateWebsocket(pex="announcing") peers = self.site.getConnectedPeers() From 325f071329eaf7abe97e6fe4d79f6cbeb8163562 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 3 Nov 2020 23:25:29 +0700 Subject: [PATCH 045/114] Fixes and refactoring in Connection.py, Peer.py --- src/Connection/Connection.py | 86 ++++++++++++++++++++++-------------- src/Peer/Peer.py | 17 ++++--- 2 files changed, 64 insertions(+), 39 deletions(-) diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 27ae3734..4a7c6ab9 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -118,6 +118,9 @@ class Connection(object): # Open connection to peer and wait for handshake def connect(self): + if not self.event_connected or self.event_connected.ready(): + self.event_connected = gevent.event.AsyncResult() + self.type = "out" if self.ip_type == "onion": if not self.server.tor_manager or not self.server.tor_manager.enabled: @@ -148,37 +151,56 @@ class Connection(object): self.sock.connect(sock_address) - # Implicit SSL - should_encrypt = not self.ip_type == "onion" and self.ip not in self.server.broken_ssl_ips and self.ip not in config.ip_local - if self.cert_pin: - self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa", cert_pin=self.cert_pin) - self.sock.do_handshake() - self.crypt = "tls-rsa" - self.sock_wrapped = True - elif should_encrypt and "tls-rsa" in CryptConnection.manager.crypt_supported: + if self.shouldEncrypt(): try: - self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa") - self.sock.do_handshake() - self.crypt = "tls-rsa" - self.sock_wrapped = True + self.wrapSocket() except Exception as err: - if not config.force_encryption: - self.log("Crypt connection error, adding %s:%s as broken ssl. %s" % (self.ip, self.port, Debug.formatException(err))) - self.server.broken_ssl_ips[self.ip] = True - self.sock.close() - self.crypt = None - self.sock = self.createSocket() - self.sock.settimeout(30) - self.sock.connect(sock_address) + if self.sock: + self.sock.close() + self.sock = None + if self.mustEncrypt(): + raise + self.log("Crypt connection error, adding %s:%s as broken ssl. %s" % (self.ip, self.port, Debug.formatException(err))) + self.server.broken_ssl_ips[self.ip] = True + return self.connect() # Detect protocol - self.send({"cmd": "handshake", "req_id": 0, "params": self.getHandshakeInfo()}) event_connected = self.event_connected + self.send({"cmd": "handshake", "req_id": 0, "params": self.getHandshakeInfo()}) gevent.spawn(self.messageLoop) connect_res = event_connected.get() # Wait for handshake - self.sock.settimeout(timeout_before) + if self.sock: + self.sock.settimeout(timeout_before) return connect_res + def mustEncrypt(self): + if self.cert_pin: + return True + if (not self.ip_type == "onion") and config.force_encryption: + return True + return False + + def shouldEncrypt(self): + if self.mustEncrypt(): + return True + return ( + (not self.ip_type == "onion") + and + (self.ip not in self.server.broken_ssl_ips) + and + (self.ip not in config.ip_local) + and + ("tls-rsa" in CryptConnection.manager.crypt_supported) + ) + + def wrapSocket(self, crypt="tls-rsa", do_handshake=True): + server = (self.type == "in") + sock = CryptConnection.manager.wrapSocket(self.sock, crypt, server=server, cert_pin=self.cert_pin) + sock.do_handshake() + self.crypt = crypt + self.sock_wrapped = True + self.sock = sock + # Handle incoming connection def handleIncomingConnection(self, sock): self.log("Incoming connection...") @@ -192,9 +214,7 @@ class Connection(object): first_byte = sock.recv(1, gevent.socket.MSG_PEEK) if first_byte == b"\x16": self.log("Crypt in connection using implicit SSL") - self.sock = CryptConnection.manager.wrapSocket(self.sock, "tls-rsa", True) - self.sock_wrapped = True - self.crypt = "tls-rsa" + self.wrapSocket(do_handshake=False) except Exception as err: self.log("Socket peek error: %s" % Debug.formatException(err)) self.messageLoop() @@ -435,7 +455,6 @@ class Connection(object): self.updateName() self.event_connected.set(True) # Mark handshake as done - self.event_connected = None self.handshake_time = time.time() # Handle incoming message @@ -459,12 +478,10 @@ class Connection(object): self.last_ping_delay = ping # Server switched to crypt, lets do it also if not crypted already if message.get("crypt") and not self.sock_wrapped: - self.crypt = message["crypt"] + crypt = message["crypt"] server = (self.type == "in") - self.log("Crypt out connection using: %s (server side: %s, ping: %.3fs)..." % (self.crypt, server, ping)) - self.sock = CryptConnection.manager.wrapSocket(self.sock, self.crypt, server, cert_pin=self.cert_pin) - self.sock.do_handshake() - self.sock_wrapped = True + self.log("Crypt out connection using: %s (server side: %s, ping: %.3fs)..." % (crypt, server, ping)) + self.wrapSocket(crypt) if not self.sock_wrapped and self.cert_pin: self.close("Crypt connection error: Socket not encrypted, but certificate pin present") @@ -492,8 +509,7 @@ class Connection(object): server = (self.type == "in") self.log("Crypt in connection using: %s (server side: %s)..." % (self.crypt, server)) try: - self.sock = CryptConnection.manager.wrapSocket(self.sock, self.crypt, server, cert_pin=self.cert_pin) - self.sock_wrapped = True + self.wrapSocket(self.crypt) except Exception as err: if not config.force_encryption: self.log("Crypt connection error, adding %s:%s as broken ssl. %s" % (self.ip, self.port, Debug.formatException(err))) @@ -640,6 +656,10 @@ class Connection(object): self.sock = None self.unpacker = None self.event_connected = None + self.crypt = None + self.sock_wrapped = False + + return True def updateOnlineStatus(self, outgoing_activity=False, successful_activity=False): self.server.updateOnlineStatus(self, diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 0a518fdc..2e809b26 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -33,6 +33,7 @@ class Peer(object): self.key = "%s:%s" % (ip, port) self.log_level = logging.DEBUG + self.connection_error_log_level = logging.DEBUG self.connection = None self.connection_server = connection_server @@ -61,8 +62,10 @@ class Peer(object): else: return getattr(self, key) - def log(self, text): - if self.log_level <= logging.DEBUG: + def log(self, text, log_level = None): + if log_level is None: + log_level = self.log_level + if log_level <= logging.DEBUG: if not config.verbose: return # Only log if we are in debug mode @@ -73,7 +76,7 @@ class Peer(object): else: logger = logging.getLogger() - logger.log(self.log_level, "%s:%s %s" % (self.ip, self.port, text)) + logger.log(log_level, "%s:%s %s" % (self.ip, self.port, text)) # Site marks its Peers protected, if it has not enough peers connected. # This is to be used to prevent disconnecting from peers when doing @@ -124,12 +127,14 @@ class Peer(object): import main connection_server = main.file_server self.connection = connection_server.getConnection(self.ip, self.port, site=self.site, is_tracker_connection=self.is_tracker_connection) - self.reputation += 1 - self.connection.sites += 1 + if self.connection and self.connection.connected: + self.reputation += 1 + self.connection.sites += 1 except Exception as err: self.onConnectionError("Getting connection error") self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" % - (Debug.formatException(err), self.connection_error, self.hash_failed)) + (Debug.formatException(err), self.connection_error, self.hash_failed), + log_level=self.connection_error_log_level) self.connection = None return self.connection From 90d01e6004544871e87abf5abbcd27c40bb44e2c Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 4 Nov 2020 09:25:44 +0700 Subject: [PATCH 046/114] Fix a tor issue introduced in the latest changes --- src/Connection/ConnectionServer.py | 1 + src/File/FileServer.py | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index b66d1739..20bd165a 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -102,6 +102,7 @@ class ConnectionServer(object): CryptConnection.manager.loadCerts() if config.tor != "disable": self.tor_manager.start() + self.tor_manager.startOnions() if not self.port: self.log.info("No port found, not binding") return False diff --git a/src/File/FileServer.py b/src/File/FileServer.py index b228680a..eea34ff0 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -260,13 +260,8 @@ class FileServer(ConnectionServer): @util.Noparallel(queue=True) def recheckPort(self): - if not self.recheck_port: - return - - if not self.port_opened: + if self.recheck_port: self.portCheck() - if not self.port_opened["ipv4"]: - self.tor_manager.startOnions() self.recheck_port = False # Returns False if Internet is immediately available @@ -322,6 +317,8 @@ class FileServer(ConnectionServer): site.invalidateUpdateTime(invalid_interval) def updateSites(self, check_files=False): + self.recheckPort() + task_nr = self.update_sites_task_next_nr self.update_sites_task_next_nr += 1 From 27ce79f04432c732d0ae9903103d75a486b804f7 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 4 Nov 2020 09:56:29 +0700 Subject: [PATCH 047/114] Fix a typo in FileServer --- src/File/FileServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index eea34ff0..33dc7ff7 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -492,7 +492,7 @@ class FileServer(ConnectionServer): continue if last_activity_time > now - threshold: continue - if len(self.update_pool) == 0: + if len(self.update_pool) != 0: continue log.info("No network activity for %.2fs. Running an update for a random site.", From 6c8b059f57b6f7787e505b14fafd9ec78aa81caa Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 4 Nov 2020 16:05:01 +0700 Subject: [PATCH 048/114] FileServer: small fixes --- src/File/FileServer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 33dc7ff7..c2a7d7d0 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -498,7 +498,7 @@ class FileServer(ConnectionServer): log.info("No network activity for %.2fs. Running an update for a random site.", now - last_activity_time ) - self.update_pool.spawn(self.updateRandomSite) + self.update_pool.spawn(self.updateRandomSite, force=True) # Periodic reloading of tracker files def reloadTrackerFilesThread(self): @@ -576,6 +576,10 @@ class FileServer(ConnectionServer): from Debug import DebugReloader DebugReloader.watcher.addCallback(self.reload) + # XXX: for initializing self.sites + # Remove this line when self.sites gets completely unused + self.getSites() + if not passive_mode: self.spawn(self.updateSites) thread_reaload_tracker_files = self.spawn(self.reloadTrackerFilesThread) From 7354d712e0b31e1eb6b497b4857fd17674926211 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 4 Nov 2020 16:08:01 +0700 Subject: [PATCH 049/114] Be more persistent in delivering site updates. --- src/File/FileRequest.py | 2 +- src/Site/Site.py | 125 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 120 insertions(+), 7 deletions(-) diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index 65c335a9..f7249d81 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -165,7 +165,7 @@ class FileRequest(object): peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update") # Add or get peer # On complete publish to other peers diffs = params.get("diffs", {}) - site.onComplete.once(lambda: site.publish(inner_path=inner_path, diffs=diffs, limit=3), "publish_%s" % inner_path) + site.onComplete.once(lambda: site.publish(inner_path=inner_path, diffs=diffs), "publish_%s" % inner_path) # Load new content file and download changed files in new thread def downloader(): diff --git a/src/Site/Site.py b/src/Site/Site.py index 1581a106..d79d16ec 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -11,6 +11,7 @@ import base64 import gevent import gevent.pool +import gevent.lock import util from Config import config @@ -61,6 +62,90 @@ class ScaledTimeoutHandler: else: return None +class BackgroundPublisher: + def __init__(self, site, published=[], limit=5, inner_path="content.json", diffs={}): + self.site = site + self.threads = gevent.pool.Pool(None) + self.inner_path = inner_path + self.stages = [ + { + "interval": ScaledTimeoutHandler(60, 60), + "max_tries": 2, + "tries": 0, + "limit": 0, + "done": False + }, + { + "interval": ScaledTimeoutHandler(60 * 10, 60 * 10), + "max_tries": 5, + "tries": 0, + "limit": 0, + "done": False + } + ] + self.reinit(published=published, limit=limit, diffs=diffs) + + def reinit(self, published=[], limit=5, diffs={}): + self.threads.kill() + self.published = published + self.diffs = diffs + + i = 0 + for stage in self.stages: + stage["nr"] = i + stage["limit"] = limit * (2 + i) + stage["tries"] = False + stage["done"] = False + stage["thread"] = None + if i > 0: + stage["interval"].done() + i += 1 + + def isStageComplete(self, stage): + if not stage["done"]: + stage["done"] = len(self.published) >= stage["limit"] + if not stage["done"]: + stage["done"] = stage["tries"] >= stage["max_tries"] + return stage["done"] + + def isComplete(self): + for stage in self.stages: + if not self.isStageComplete(stage): + return False + return True + + def process(self): + for stage in self.stages: + if not self.isStageComplete(stage): + self.processStage(stage) + break + return self.isComplete() + + def processStage(self, stage): + if not stage["interval"].isExpired(0): + return + + if len(self.site.peers) < stage["limit"]: + self.site.announce(mode="more") + + if not stage["thread"]: + peers = list(self.site.peers.values()) + random.shuffle(peers) + stage["thread"] = self.threads.spawn(self.site.publisher, + self.inner_path, peers, self.published, stage["limit"], diffs=self.diffs, max_retries=1) + + stage["tries"] += 1 + stage["interval"].done() + + self.site.log.info("Background publisher: Stage #%s: %s published to %s/%s peers", + stage["nr"], self.inner_path, len(self.published), stage["limit"]) + + def finalize(self): + self.threads.kill() + self.site.log.info("Background publisher: Published %s to %s peers", self.inner_path, len(self.published)) + + + @PluginManager.acceptPlugins class Site(object): @@ -81,6 +166,9 @@ class Site(object): scaler=self.getActivityRating) ] + self.background_publishers = {} + self.background_publishers_lock = gevent.lock.RLock() + self.content = None # Load content.json self.peers = {} # Key: ip:port, Value: Peer.Peer self.peers_recent = collections.deque(maxlen=150) @@ -328,6 +416,11 @@ class Site(object): inner_path, time.time() - s, len(self.worker_manager.tasks) )) + + # If no file tasks have been started, worker_manager.checkComplete() + # never called. So call it explicitly. + self.greenlet_manager.spawn(self.worker_manager.checkComplete) + return True # Return bad files with less than 3 retry @@ -662,7 +755,7 @@ class Site(object): gevent.joinall(content_threads) # Publish worker - def publisher(self, inner_path, peers, published, limit, diffs={}, event_done=None, cb_progress=None): + def publisher(self, inner_path, peers, published, limit, diffs={}, event_done=None, cb_progress=None, max_retries=2): file_size = self.storage.getSize(inner_path) content_json_modified = self.content_manager.contents[inner_path]["modified"] body = self.storage.read(inner_path) @@ -687,7 +780,7 @@ class Site(object): timeout = 10 + int(file_size / 1024) result = {"exception": "Timeout"} - for retry in range(2): + for retry in range(max_retries): try: with gevent.Timeout(timeout, False): result = peer.publish(self.address, inner_path, body, content_json_modified, diffs) @@ -708,6 +801,25 @@ class Site(object): self.log.info("[FAILED] %s: %s" % (peer.key, result)) time.sleep(0.01) + def addBackgroundPublisher(self, published=[], limit=5, inner_path="content.json", diffs={}): + with self.background_publishers_lock: + if self.background_publishers.get(inner_path, None): + background_publisher = self.background_publishers[inner_path] + background_publisher.reinit(published=published, limit=limit, diffs=diffs) + else: + background_publisher = BackgroundPublisher(self, published=published, limit=limit, inner_path=inner_path, diffs=diffs) + self.background_publishers[inner_path] = background_publisher + + gevent.spawn(background_publisher.process) + + def processBackgroundPublishers(self): + with self.background_publishers_lock: + for inner_path, background_publisher in list(self.background_publishers.items()): + background_publisher.process() + if background_publisher.isComplete(): + background_publisher.finalize() + del self.background_publishers[inner_path] + # Update content.json on peers @util.Noparallel() def publish(self, limit="default", inner_path="content.json", diffs={}, cb_progress=None): @@ -752,12 +864,11 @@ class Site(object): # Publish more peers in the backgroup self.log.info( - "Published %s to %s peers, publishing to %s more peers in the background" % - (inner_path, len(published), limit) + "Published %s to %s peers, publishing to more peers in the background" % + (inner_path, len(published)) ) - for thread in range(2): - gevent.spawn(self.publisher, inner_path, peers, published, limit=limit * 2, diffs=diffs) + self.addBackgroundPublisher(published=published, limit=limit, inner_path=inner_path, diffs=diffs) # Send my hashfield to every connected peer if changed gevent.spawn(self.sendMyHashfield, 100) @@ -1296,6 +1407,8 @@ class Site(object): with gevent.Timeout(10, exception=False): self.announcer.announcePex() + self.processBackgroundPublishers() + self.update() return True From 5d5b3684cc3e2875673a6307451388a2e19f0079 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 4 Nov 2020 18:22:14 +0700 Subject: [PATCH 050/114] ContentManager: split verifyFile() into 2 functions and always log the verify error at INFO level --- src/Content/ContentManager.py | 191 +++++++++++++++++----------------- 1 file changed, 98 insertions(+), 93 deletions(-) diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index 27da402b..5fe80952 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -925,102 +925,107 @@ class ContentManager(object): return True # All good + def verifyContentJson(self, inner_path, file, ignore_same=True): + from Crypt import CryptBitcoin + + if type(file) is dict: + new_content = file + else: + try: + if sys.version_info.major == 3 and sys.version_info.minor < 6: + new_content = json.loads(file.read().decode("utf8")) + else: + new_content = json.load(file) + except Exception as err: + raise VerifyError("Invalid json file: %s" % err) + if inner_path in self.contents: + old_content = self.contents.get(inner_path, {"modified": 0}) + # Checks if its newer the ours + if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json + return None + elif old_content["modified"] > new_content["modified"]: # We have newer + raise VerifyError( + "We have newer (Our: %s, Sent: %s)" % + (old_content["modified"], new_content["modified"]) + ) + if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+) + raise VerifyError("Modify timestamp is in the far future!") + if self.isArchived(inner_path, new_content["modified"]): + if inner_path in self.site.bad_files: + del self.site.bad_files[inner_path] + raise VerifyError("This file is archived!") + # Check sign + sign = new_content.get("sign") + signs = new_content.get("signs", {}) + if "sign" in new_content: + del(new_content["sign"]) # The file signed without the sign + if "signs" in new_content: + del(new_content["signs"]) # The file signed without the signs + + sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace + + # Fix float representation error on Android + modified = new_content["modified"] + if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"): + modified_fixed = "{:.6f}".format(modified).strip("0.") + sign_content = sign_content.replace( + '"modified": %s' % repr(modified), + '"modified": %s' % modified_fixed + ) + + if signs: # New style signing + valid_signers = self.getValidSigners(inner_path, new_content) + signs_required = self.getSignsRequired(inner_path, new_content) + + if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json + signers_data = "%s:%s" % (signs_required, ",".join(valid_signers)) + if not CryptBitcoin.verify(signers_data, self.site.address, new_content["signers_sign"]): + raise VerifyError("Invalid signers_sign!") + + if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid + raise VerifyError("Invalid cert!") + + valid_signs = 0 + for address in valid_signers: + if address in signs: + valid_signs += CryptBitcoin.verify(sign_content, address, signs[address]) + if valid_signs >= signs_required: + break # Break if we has enough signs + if valid_signs < signs_required: + raise VerifyError("Valid signs: %s/%s" % (valid_signs, signs_required)) + else: + return self.verifyContent(inner_path, new_content) + else: # Old style signing + raise VerifyError("Invalid old-style sign") + + def verifyOrdinaryFile(self, inner_path, file, ignore_same=True): + file_info = self.getFileInfo(inner_path) + if file_info: + if CryptHash.sha512sum(file) != file_info.get("sha512", ""): + raise VerifyError("Invalid hash") + + if file_info.get("size", 0) != file.tell(): + raise VerifyError( + "File size does not match %s <> %s" % + (inner_path, file.tell(), file_info.get("size", 0)) + ) + + return True + + else: # File not in content.json + raise VerifyError("File not in content.json") + # Verify file validity # Return: None = Same as before, False = Invalid, True = Valid def verifyFile(self, inner_path, file, ignore_same=True): - if inner_path.endswith("content.json"): # content.json: Check using sign - from Crypt import CryptBitcoin - try: - if type(file) is dict: - new_content = file - else: - try: - if sys.version_info.major == 3 and sys.version_info.minor < 6: - new_content = json.loads(file.read().decode("utf8")) - else: - new_content = json.load(file) - except Exception as err: - raise VerifyError("Invalid json file: %s" % err) - if inner_path in self.contents: - old_content = self.contents.get(inner_path, {"modified": 0}) - # Checks if its newer the ours - if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json - return None - elif old_content["modified"] > new_content["modified"]: # We have newer - raise VerifyError( - "We have newer (Our: %s, Sent: %s)" % - (old_content["modified"], new_content["modified"]) - ) - if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+) - raise VerifyError("Modify timestamp is in the far future!") - if self.isArchived(inner_path, new_content["modified"]): - if inner_path in self.site.bad_files: - del self.site.bad_files[inner_path] - raise VerifyError("This file is archived!") - # Check sign - sign = new_content.get("sign") - signs = new_content.get("signs", {}) - if "sign" in new_content: - del(new_content["sign"]) # The file signed without the sign - if "signs" in new_content: - del(new_content["signs"]) # The file signed without the signs - - sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace - - # Fix float representation error on Android - modified = new_content["modified"] - if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"): - modified_fixed = "{:.6f}".format(modified).strip("0.") - sign_content = sign_content.replace( - '"modified": %s' % repr(modified), - '"modified": %s' % modified_fixed - ) - - if signs: # New style signing - valid_signers = self.getValidSigners(inner_path, new_content) - signs_required = self.getSignsRequired(inner_path, new_content) - - if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json - signers_data = "%s:%s" % (signs_required, ",".join(valid_signers)) - if not CryptBitcoin.verify(signers_data, self.site.address, new_content["signers_sign"]): - raise VerifyError("Invalid signers_sign!") - - if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid - raise VerifyError("Invalid cert!") - - valid_signs = 0 - for address in valid_signers: - if address in signs: - valid_signs += CryptBitcoin.verify(sign_content, address, signs[address]) - if valid_signs >= signs_required: - break # Break if we has enough signs - if valid_signs < signs_required: - raise VerifyError("Valid signs: %s/%s" % (valid_signs, signs_required)) - else: - return self.verifyContent(inner_path, new_content) - else: # Old style signing - raise VerifyError("Invalid old-style sign") - - except Exception as err: - self.log.warning("%s: verify sign error: %s" % (inner_path, Debug.formatException(err))) - raise err - - else: # Check using sha512 hash - file_info = self.getFileInfo(inner_path) - if file_info: - if CryptHash.sha512sum(file) != file_info.get("sha512", ""): - raise VerifyError("Invalid hash") - - if file_info.get("size", 0) != file.tell(): - raise VerifyError( - "File size does not match %s <> %s" % - (inner_path, file.tell(), file_info.get("size", 0)) - ) - - return True - - else: # File not in content.json - raise VerifyError("File not in content.json") + try: + if inner_path.endswith("content.json"): + return self.verifyContentJson(inner_path, file, ignore_same) + else: + return self.verifyOrdinaryFile(inner_path, file, ignore_same) + except Exception as err: + self.log.info("%s: verify error: %s" % (inner_path, Debug.formatException(err))) + raise err def optionalDelete(self, inner_path): self.site.storage.delete(inner_path) From 570f854485aa8e7ad7e9fc3045b4226c95aeffcc Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 4 Nov 2020 21:57:54 +0700 Subject: [PATCH 051/114] Fix a typo in Site.py --- src/Site/Site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index d79d16ec..5e15172b 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -94,7 +94,7 @@ class BackgroundPublisher: for stage in self.stages: stage["nr"] = i stage["limit"] = limit * (2 + i) - stage["tries"] = False + stage["tries"] = 0 stage["done"] = False stage["thread"] = None if i > 0: From 697b12d808d42f93865c754ed0bea57aa99b12ec Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 4 Nov 2020 22:13:21 +0700 Subject: [PATCH 052/114] Fix naming. verifyContentJson() should be verifyContentFile() --- src/Content/ContentManager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index 5fe80952..2283c91b 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -925,7 +925,7 @@ class ContentManager(object): return True # All good - def verifyContentJson(self, inner_path, file, ignore_same=True): + def verifyContentFile(self, inner_path, file, ignore_same=True): from Crypt import CryptBitcoin if type(file) is dict: @@ -1020,7 +1020,7 @@ class ContentManager(object): def verifyFile(self, inner_path, file, ignore_same=True): try: if inner_path.endswith("content.json"): - return self.verifyContentJson(inner_path, file, ignore_same) + return self.verifyContentFile(inner_path, file, ignore_same) else: return self.verifyOrdinaryFile(inner_path, file, ignore_same) except Exception as err: From 4e27e300e3e449714bd861967b1d30cb96f0d3cd Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 23 Mar 2021 16:01:36 +0700 Subject: [PATCH 053/114] Delete plugin/AnnounceShare/ --- plugins/AnnounceShare/AnnounceSharePlugin.py | 190 ------------------ .../AnnounceShare/Test/TestAnnounceShare.py | 24 --- plugins/AnnounceShare/Test/conftest.py | 3 - plugins/AnnounceShare/Test/pytest.ini | 5 - plugins/AnnounceShare/__init__.py | 1 - plugins/AnnounceShare/plugin_info.json | 5 - 6 files changed, 228 deletions(-) delete mode 100644 plugins/AnnounceShare/AnnounceSharePlugin.py delete mode 100644 plugins/AnnounceShare/Test/TestAnnounceShare.py delete mode 100644 plugins/AnnounceShare/Test/conftest.py delete mode 100644 plugins/AnnounceShare/Test/pytest.ini delete mode 100644 plugins/AnnounceShare/__init__.py delete mode 100644 plugins/AnnounceShare/plugin_info.json diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py deleted file mode 100644 index 057ce55a..00000000 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ /dev/null @@ -1,190 +0,0 @@ -import time -import os -import logging -import json -import atexit - -import gevent - -from Config import config -from Plugin import PluginManager -from util import helper - - -class TrackerStorage(object): - def __init__(self): - self.log = logging.getLogger("TrackerStorage") - self.file_path = "%s/trackers.json" % config.data_dir - self.load() - self.time_discover = 0.0 - atexit.register(self.save) - - def getDefaultFile(self): - return {"shared": {}} - - def onTrackerFound(self, tracker_address, type="shared", my=False): - if not tracker_address.startswith("zero://"): - return False - - trackers = self.getTrackers() - added = False - if tracker_address not in trackers: - trackers[tracker_address] = { - "time_added": time.time(), - "time_success": 0, - "latency": 99.0, - "num_error": 0, - "my": False - } - self.log.debug("New tracker found: %s" % tracker_address) - added = True - - trackers[tracker_address]["time_found"] = time.time() - trackers[tracker_address]["my"] = my - return added - - def onTrackerSuccess(self, tracker_address, latency): - trackers = self.getTrackers() - if tracker_address not in trackers: - return False - - trackers[tracker_address]["latency"] = latency - trackers[tracker_address]["time_success"] = time.time() - trackers[tracker_address]["num_error"] = 0 - - def onTrackerError(self, tracker_address): - trackers = self.getTrackers() - if tracker_address not in trackers: - return False - - trackers[tracker_address]["time_error"] = time.time() - trackers[tracker_address]["num_error"] += 1 - - if len(self.getWorkingTrackers()) >= config.working_shared_trackers_limit: - error_limit = 5 - else: - error_limit = 30 - error_limit - - if trackers[tracker_address]["num_error"] > error_limit and trackers[tracker_address]["time_success"] < time.time() - 60 * 60: - self.log.debug("Tracker %s looks down, removing." % tracker_address) - del trackers[tracker_address] - - def getTrackers(self, type="shared"): - return self.file_content.setdefault(type, {}) - - def getWorkingTrackers(self, type="shared"): - trackers = { - key: tracker for key, tracker in self.getTrackers(type).items() - if tracker["time_success"] > time.time() - 60 * 60 - } - return trackers - - def getFileContent(self): - if not os.path.isfile(self.file_path): - open(self.file_path, "w").write("{}") - return self.getDefaultFile() - try: - return json.load(open(self.file_path)) - except Exception as err: - self.log.error("Error loading trackers list: %s" % err) - return self.getDefaultFile() - - def load(self): - self.file_content = self.getFileContent() - - trackers = self.getTrackers() - self.log.debug("Loaded %s shared trackers" % len(trackers)) - for address, tracker in list(trackers.items()): - tracker["num_error"] = 0 - if not address.startswith("zero://"): - del trackers[address] - - def save(self): - s = time.time() - helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True).encode("utf8")) - self.log.debug("Saved in %.3fs" % (time.time() - s)) - - def discoverTrackers(self, peers): - if len(self.getWorkingTrackers()) > config.working_shared_trackers_limit: - return False - s = time.time() - num_success = 0 - for peer in peers: - if peer.connection and peer.connection.handshake.get("rev", 0) < 3560: - continue # Not supported - - res = peer.request("getTrackers") - if not res or "error" in res: - continue - - num_success += 1 - for tracker_address in res["trackers"]: - if type(tracker_address) is bytes: # Backward compatibilitys - tracker_address = tracker_address.decode("utf8") - added = self.onTrackerFound(tracker_address) - if added: # Only add one tracker from one source - break - - if not num_success and len(peers) < 20: - self.time_discover = 0.0 - - if num_success: - self.save() - - self.log.debug("Trackers discovered from %s/%s peers in %.3fs" % (num_success, len(peers), time.time() - s)) - - -if "tracker_storage" not in locals(): - tracker_storage = TrackerStorage() - - -@PluginManager.registerTo("SiteAnnouncer") -class SiteAnnouncerPlugin(object): - def getTrackers(self): - if tracker_storage.time_discover < time.time() - 5 * 60: - tracker_storage.time_discover = time.time() - gevent.spawn(tracker_storage.discoverTrackers, self.site.getConnectedPeers()) - trackers = super(SiteAnnouncerPlugin, self).getTrackers() - shared_trackers = list(tracker_storage.getTrackers("shared").keys()) - if shared_trackers: - return trackers + shared_trackers - else: - return trackers - - def announceTracker(self, tracker, *args, **kwargs): - res = super(SiteAnnouncerPlugin, self).announceTracker(tracker, *args, **kwargs) - if res: - latency = res - tracker_storage.onTrackerSuccess(tracker, latency) - elif res is False: - tracker_storage.onTrackerError(tracker) - - return res - - -@PluginManager.registerTo("FileRequest") -class FileRequestPlugin(object): - def actionGetTrackers(self, params): - shared_trackers = list(tracker_storage.getWorkingTrackers("shared").keys()) - self.response({"trackers": shared_trackers}) - - -@PluginManager.registerTo("FileServer") -class FileServerPlugin(object): - def portCheck(self, *args, **kwargs): - res = super(FileServerPlugin, self).portCheck(*args, **kwargs) - if res and not config.tor == "always" and "Bootstrapper" in PluginManager.plugin_manager.plugin_names: - for ip in self.ip_external_list: - my_tracker_address = "zero://%s:%s" % (ip, config.fileserver_port) - tracker_storage.onTrackerFound(my_tracker_address, my=True) - return res - - -@PluginManager.registerTo("ConfigPlugin") -class ConfigPlugin(object): - def createArguments(self): - group = self.parser.add_argument_group("AnnounceShare plugin") - group.add_argument('--working_shared_trackers_limit', help='Stop discovering new shared trackers after this number of shared trackers reached', default=5, type=int, metavar='limit') - - return super(ConfigPlugin, self).createArguments() diff --git a/plugins/AnnounceShare/Test/TestAnnounceShare.py b/plugins/AnnounceShare/Test/TestAnnounceShare.py deleted file mode 100644 index 7178eac8..00000000 --- a/plugins/AnnounceShare/Test/TestAnnounceShare.py +++ /dev/null @@ -1,24 +0,0 @@ -import pytest - -from AnnounceShare import AnnounceSharePlugin -from Peer import Peer -from Config import config - - -@pytest.mark.usefixtures("resetSettings") -@pytest.mark.usefixtures("resetTempSettings") -class TestAnnounceShare: - def testAnnounceList(self, file_server): - open("%s/trackers.json" % config.data_dir, "w").write("{}") - tracker_storage = AnnounceSharePlugin.tracker_storage - tracker_storage.load() - peer = Peer(file_server.ip, 1544, connection_server=file_server) - assert peer.request("getTrackers")["trackers"] == [] - - tracker_storage.onTrackerFound("zero://%s:15441" % file_server.ip) - assert peer.request("getTrackers")["trackers"] == [] - - # It needs to have at least one successfull announce to be shared to other peers - tracker_storage.onTrackerSuccess("zero://%s:15441" % file_server.ip, 1.0) - assert peer.request("getTrackers")["trackers"] == ["zero://%s:15441" % file_server.ip] - diff --git a/plugins/AnnounceShare/Test/conftest.py b/plugins/AnnounceShare/Test/conftest.py deleted file mode 100644 index 5abd4dd6..00000000 --- a/plugins/AnnounceShare/Test/conftest.py +++ /dev/null @@ -1,3 +0,0 @@ -from src.Test.conftest import * - -from Config import config diff --git a/plugins/AnnounceShare/Test/pytest.ini b/plugins/AnnounceShare/Test/pytest.ini deleted file mode 100644 index d09210d1..00000000 --- a/plugins/AnnounceShare/Test/pytest.ini +++ /dev/null @@ -1,5 +0,0 @@ -[pytest] -python_files = Test*.py -addopts = -rsxX -v --durations=6 -markers = - webtest: mark a test as a webtest. \ No newline at end of file diff --git a/plugins/AnnounceShare/__init__.py b/plugins/AnnounceShare/__init__.py deleted file mode 100644 index dc1e40bd..00000000 --- a/plugins/AnnounceShare/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import AnnounceSharePlugin diff --git a/plugins/AnnounceShare/plugin_info.json b/plugins/AnnounceShare/plugin_info.json deleted file mode 100644 index 0ad07e71..00000000 --- a/plugins/AnnounceShare/plugin_info.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "AnnounceShare", - "description": "Share possible trackers between clients.", - "default": "enabled" -} \ No newline at end of file From 3677684971580aab1ddd7d0507d5f9937a965b7d Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 23 Mar 2021 19:02:43 +0700 Subject: [PATCH 054/114] Add ZNE-ChangeLog/ChangeLog-0.8.0.md --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 ZNE-ChangeLog/ChangeLog-0.8.0.md diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md new file mode 100644 index 00000000..9925dbdc --- /dev/null +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -0,0 +1,60 @@ + +## Core + +**Network:** + +* Reworked the algorithm of checking zite updates on startup / after the network outages / periodically. ZeroNet tries not to spam too many update queries at once in order to prevent network overload. (Which especially the issue when running over Tor.) At the same time, it tries to keep balance between frequent checks for frequently updating zites and ensuring that all the zites are checked in some reasonable time interval. Tests show that the full check cycle for a peer that hosts 800+ zites and is connected over Tor can take up to several hours. We cannot significantly reduce this time, since the Tor throughput is the bottleneck. Running more checks at once just results in more connections to fail. The other bottleneck is the HDD throughput. Increasing the parallelization doesn't help in this case as well. So the implemented solution **decreases** the concurency. +* Improved the Internet outage detection and the recovery procedures after the Internet be back. ZeroNet "steps back" and schedules rechecking zites that were checked shortly before the Internet connection get lost. The network outage detection normally has some lag, so the recently checked zites are better to checked again. +* When the network is down, reduce the frequency of connection attempts to prevent overloading Tor with hanged connections. +* The connection handling code had several bugs that were hidden by silently ignored exceptions. These were fixed, but some new ones may be introduced. +* For each zite the activity rate is calculated based on the last modification time. The milestone values are 1 hour, 5 hours, 24 hours, 3 days and 7 days. The activity rate is used to scale frequency of various maintenance routines, including update checks, reannounces, dead connections check etc. +* The activity rate is also used to calculate the minimum preferred number of active connections per each zite. +* The reannounce frequency is adjusted dynamically based on: + * Activity. More activity ==> frequent announces. + * Peer count. More peers ==> rare announces. + * Tracker count. More trackers ==> frequent announces to iterate over more trackers. +* For owned zites, the activity rate doesn't drop below 0.6 to force more frequent checks. This, however, can be used to investigate which peer belongs to the zire owner. A new commnd line option `--expose_no_ownership` is introduced to disable that behavior. +* `ZeroNet` now tries harder in delivering updates to more peers in the background. + +**Other:** + +* Implemented the log level overriding for separate modules for easier debugging. + +## Plugins + +### TrackerShare + +The `AnnounceShare` plugin renamed to `TrackerShare` and redesigned. The new name is more consistent and better reflects the purpose of the plugin: sharing the list of known working trackers. Other `Announce`-like names are clearly related to zite announcing facilities: `AnnounceBitTorrent` (announce to a BitTorrent tracker), `AnnounceLocal` (announce in a Local Area Network), `AnnounceZero` (announce to a Zero tracker). + +Changes in the plugin: + +* The default total tracker limit increased up to 20 from 5. This reduces the probability of accidental splitting the network into segments that operate with disjoint sets of trackers. +* The plugin now shares any types of trackers, working both on IP and Onion networks, not limiting solely to Zero-based IP trackers. +* The plugin now takes into account not only the total number of known working trackers, but also does it on per protocol basis. The defaults are to keep 10 trackers for Zero protocol and 5 trackers per each other protocol. The available protocols are detected automatically. (For example, UDP is considered disabled, when working in the Tor-always mode.) The following protocols are recognized: `zero://` (Zero Tracker), `udp://` (UDP-based bitTorrent tracker), `http://` or `https://` (HTTP or HTTPS-based bitTorrent tracker; considered the same protocol). In case of new protocols implemented by third-party plugins, trackers are grouped automatically by the protocol prefix. +* Reworked the interaction of this plugin and the Zero tracker (`Bootstrapper`) plugin. Previously: `AnnounceShare` checks if `Bootstrapper` is running and adds its addresses to the list. Now: the tracker explicitly asks `TrackerShare` to add its addresses. +* The plugin allows adjustings the behaviour per each tracker entry with the following boolean fields: + * `my` — "My" trackers get never deleted on announce errors, but aren't saved between restarts. Designed to be used by tracker implementation plugins. `TrackerShare` acts more persistently in recommending "my" trackers to other peers. + * `persistent` — "Persistent" trackers get never deleted, when unresponsive. + * `private` — "Private" trackers are never exposed to other peer in response of the getTrackers command. + +### TrackerZero + +`TrackerZero` is an attempt of implementing the better version of the `Bootstrapper` plugin to make setting up and launching a new tracker possible in a couple of mouse clicks. Work in progress, so no detailed change log yet. The `Bootstrapper` plugin itself is kept untouched. + +The plugin has the following self-explanatory options: + +``` +enable +enable_only_in_tor_always_mode +listen_on_public_ips +listen_on_temporary_onion_address +listen_on_persistent_onion_address +``` + +Running over TOR doesn't seem to be stable so far. Any investigations and bug reports are welcome. + +### TrackerList + +`TrackerZero` is a new plugin for fetching tracker lists. By default is list is fetched from [https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt](https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt). + +TODO: add support of fetching from ZeroNet URLs From 6c8849139fa0f975b5fa5291cdbd9148eda91a3f Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 23 Mar 2021 19:05:51 +0700 Subject: [PATCH 055/114] ZNE-ChangeLog/ChangeLog-0.8.0.md: typos --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md index 9925dbdc..59f0af55 100644 --- a/ZNE-ChangeLog/ChangeLog-0.8.0.md +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -6,8 +6,8 @@ * Reworked the algorithm of checking zite updates on startup / after the network outages / periodically. ZeroNet tries not to spam too many update queries at once in order to prevent network overload. (Which especially the issue when running over Tor.) At the same time, it tries to keep balance between frequent checks for frequently updating zites and ensuring that all the zites are checked in some reasonable time interval. Tests show that the full check cycle for a peer that hosts 800+ zites and is connected over Tor can take up to several hours. We cannot significantly reduce this time, since the Tor throughput is the bottleneck. Running more checks at once just results in more connections to fail. The other bottleneck is the HDD throughput. Increasing the parallelization doesn't help in this case as well. So the implemented solution **decreases** the concurency. * Improved the Internet outage detection and the recovery procedures after the Internet be back. ZeroNet "steps back" and schedules rechecking zites that were checked shortly before the Internet connection get lost. The network outage detection normally has some lag, so the recently checked zites are better to checked again. * When the network is down, reduce the frequency of connection attempts to prevent overloading Tor with hanged connections. -* The connection handling code had several bugs that were hidden by silently ignored exceptions. These were fixed, but some new ones may be introduced. -* For each zite the activity rate is calculated based on the last modification time. The milestone values are 1 hour, 5 hours, 24 hours, 3 days and 7 days. The activity rate is used to scale frequency of various maintenance routines, including update checks, reannounces, dead connections check etc. +* The connection handling code had several bugs that were hidden by silently ignored exceptions. These were fixed, but some new ones might be introduced. +* For each zite the activity rate is calculated based on the last modification time. The milestone values are 1 hour, 5 hours, 24 hours, 3 days and 7 days. The activity rate is used to scale frequency of various maintenance routines, including update checks, reannounces, dead connection checks etc. * The activity rate is also used to calculate the minimum preferred number of active connections per each zite. * The reannounce frequency is adjusted dynamically based on: * Activity. More activity ==> frequent announces. From 1144964062e9494139dc93526dd2de943fe13cc7 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 23 Mar 2021 23:12:54 +0700 Subject: [PATCH 056/114] Make the site block check usable from plugins and core modules Fixes https://github.com/HelloZeroNet/ZeroNet/issues/1888 --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 1 + plugins/ContentFilter/ContentFilterPlugin.py | 8 ++++++++ src/Site/SiteManager.py | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md index 59f0af55..f87f1cb9 100644 --- a/ZNE-ChangeLog/ChangeLog-0.8.0.md +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -19,6 +19,7 @@ **Other:** * Implemented the log level overriding for separate modules for easier debugging. +* Make the site block check implemented in `ContentFilter` usable from plugins and core modules via `SiteManager.isAddressBlocked()`. ## Plugins diff --git a/plugins/ContentFilter/ContentFilterPlugin.py b/plugins/ContentFilter/ContentFilterPlugin.py index f2f84b49..6bd8c7f7 100644 --- a/plugins/ContentFilter/ContentFilterPlugin.py +++ b/plugins/ContentFilter/ContentFilterPlugin.py @@ -24,6 +24,14 @@ class SiteManagerPlugin(object): super(SiteManagerPlugin, self).load(*args, **kwargs) filter_storage = ContentFilterStorage(site_manager=self) + def isAddressBlocked(self, address): + # FIXME: code duplication of isSiteblocked(address) or isSiteblocked(address_hashed) + # in several places here and below + address_hashed = filter_storage.getSiteAddressHashed(address) + if filter_storage.isSiteblocked(address) or filter_storage.isSiteblocked(address_hashed): + return True + return super(SiteManagerPlugin, self).isAddressBlocked(address) + def add(self, address, *args, **kwargs): should_ignore_block = kwargs.get("ignore_block") or kwargs.get("settings") if should_ignore_block: diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 684d69fc..1b065e93 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -155,6 +155,10 @@ class SiteManager(object): def resolveDomainCached(self, domain): return self.resolveDomain(domain) + # Checks if the address is blocked. To be implemented in content filter plugins. + def isAddressBlocked(self, address): + return False + # Return: Site object or None if not found def get(self, address): if self.isDomainCached(address): From d0069471b8bcf0a61b6c5a9849621f255580065a Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 24 Mar 2021 08:04:23 +0700 Subject: [PATCH 057/114] Don't raise VerifyError with misleading message "Invalid old-style sign" when the file has no sign at all. --- src/Content/ContentManager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index 2283c91b..c5ce1d61 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -995,8 +995,10 @@ class ContentManager(object): raise VerifyError("Valid signs: %s/%s" % (valid_signs, signs_required)) else: return self.verifyContent(inner_path, new_content) - else: # Old style signing + elif sign: # Old style signing raise VerifyError("Invalid old-style sign") + else: + raise VerifyError("Not signed") def verifyOrdinaryFile(self, inner_path, file, ignore_same=True): file_info = self.getFileInfo(inner_path) From 0151546329ffa8ada1f4c09efab921560153780d Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 24 Mar 2021 09:51:07 +0700 Subject: [PATCH 058/114] ContentManager.py: move duplicated code to new method serializeForSigning() --- src/Content/ContentManager.py | 42 +++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index c5ce1d61..7d1263ef 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -651,6 +651,25 @@ class ContentManager(object): ) return files_node, files_optional_node + def serializeForSigning(self, content): + if "sign" in content: + del(content["sign"]) # The file signed without the sign + if "signs" in content: + del(content["signs"]) # The file signed without the signs + + sign_content = json.dumps(content, sort_keys=True) # Dump the json to string to remove whitespaces + + # Fix float representation error on Android + modified = content["modified"] + if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"): + modified_fixed = "{:.6f}".format(modified).strip("0.") + sign_content = sign_content.replace( + '"modified": %s' % repr(modified), + '"modified": %s' % modified_fixed + ) + + return sign_content + # Create and sign a content.json # Return: The new content if filewrite = False def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None, remove_missing_optional=False): @@ -756,12 +775,7 @@ class ContentManager(object): self.log.info("Signing %s..." % inner_path) - if "signs" in new_content: - del(new_content["signs"]) # Delete old signs - if "sign" in new_content: - del(new_content["sign"]) # Delete old sign (backward compatibility) - - sign_content = json.dumps(new_content, sort_keys=True) + sign_content = self.serializeForSigning(new_content) sign = CryptBitcoin.sign(sign_content, privatekey) # new_content["signs"] = content.get("signs", {}) # TODO: Multisig if sign: # If signing is successful (not an old address) @@ -957,21 +971,7 @@ class ContentManager(object): # Check sign sign = new_content.get("sign") signs = new_content.get("signs", {}) - if "sign" in new_content: - del(new_content["sign"]) # The file signed without the sign - if "signs" in new_content: - del(new_content["signs"]) # The file signed without the signs - - sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace - - # Fix float representation error on Android - modified = new_content["modified"] - if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"): - modified_fixed = "{:.6f}".format(modified).strip("0.") - sign_content = sign_content.replace( - '"modified": %s' % repr(modified), - '"modified": %s' % modified_fixed - ) + sign_content = self.serializeForSigning(new_content) if signs: # New style signing valid_signers = self.getValidSigners(inner_path, new_content) From 986dedfa7f4ea8e0c1cc6c4bbfe3fccdf15640c6 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 25 Mar 2021 12:41:53 +0700 Subject: [PATCH 059/114] Trying to fix incomplete updates Partially fixes https://github.com/HelloZeroNet/ZeroNet/issues/2476 Approx. every 10th update check is now performed with `since = 0` --- src/Site/Site.py | 111 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 78 insertions(+), 33 deletions(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index 5e15172b..0a8be32d 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -169,6 +169,12 @@ class Site(object): self.background_publishers = {} self.background_publishers_lock = gevent.lock.RLock() + # FZS = forced zero "since" + self.fzs_range = 20 + self.fzs_interval = 30 * 60 + self.fzs_count = random.randint(0, self.fzs_range / 4) + self.fzs_timestamp = 0 + self.content = None # Load content.json self.peers = {} # Key: ip:port, Value: Peer.Peer self.peers_recent = collections.deque(maxlen=150) @@ -530,14 +536,18 @@ class Site(object): self.log.debug("Ended downloadFile pool len: %s, skipped: %s" % (len(inner_paths), num_skipped)) # Update worker, try to find client that supports listModifications command - def updater(self, peers_try, queried, since): + def updater(self, peers_try, queried, need_queries, since): threads = [] while 1: - if not peers_try or len(queried) >= 3: # Stop after 3 successful query + if not peers_try or len(queried) >= need_queries: # Stop after 3 successful query break peer = peers_try.pop(0) + + if peer in queried: + continue + if config.verbose: - self.log.debug("CheckModifications: Try to get updates from: %s Left: %s" % (peer, peers_try)) + self.log.debug("CheckModifications: Trying to get updates from: %s Left: %s" % (peer, peers_try)) res = None with gevent.Timeout(20, exception=False): @@ -548,6 +558,8 @@ class Site(object): queried.append(peer) modified_contents = [] + send_back = [] + send_back_limit = 5 my_modified = self.content_manager.listModified(since) num_old_files = 0 for inner_path, modified in res["modified_files"].items(): # Check if the peer has newer files than we @@ -558,26 +570,56 @@ class Site(object): # We dont have this file or we have older modified_contents.append(inner_path) self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 - if has_older and num_old_files < 5: - num_old_files += 1 - self.log.debug("CheckModifications: %s client has older version of %s, publishing there (%s/5)..." % (peer, inner_path, num_old_files)) - gevent.spawn(self.publisher, inner_path, [peer], [], 1) + if has_older: + send_back.append(inner_path) + if modified_contents: - self.log.debug("CheckModifications: %s new modified file from %s" % (len(modified_contents), peer)) + self.log.info("CheckModifications: %s new modified files from %s" % (len(modified_contents), peer)) modified_contents.sort(key=lambda inner_path: 0 - res["modified_files"][inner_path]) # Download newest first + for inner_path in modified_contents: + self.log.info("CheckModifications: %s: %s > %s" % ( + inner_path, res["modified_files"][inner_path], my_modified.get(inner_path, 0) + )) t = gevent.spawn(self.pooledDownloadContent, modified_contents, only_if_bad=True) threads.append(t) - if config.verbose: - self.log.debug("CheckModifications: Waiting for %s pooledDownloadContent" % len(threads)) + + if send_back: + self.log.info("CheckModifications: %s has older versions of %s files" % (peer, len(send_back))) + if len(send_back) > send_back_limit: + self.log.info("CheckModifications: choosing %s files to publish back" % (send_back_limit)) + random.shuffle(send_back) + send_back = send_back[0:send_back_limit] + for inner_path in send_back: + self.log.info("CheckModifications: %s: %s < %s" % ( + inner_path, res["modified_files"][inner_path], my_modified.get(inner_path, 0) + )) + gevent.spawn(self.publisher, inner_path, [peer], [], 1) + + self.log.debug("CheckModifications: Waiting for %s pooledDownloadContent" % len(threads)) gevent.joinall(threads) + # We need, with some rate, to perform the full check of modifications, + # "since the beginning of time", instead of the partial one. + def getForcedZeroSince(self): + now = time.time() + if self.fzs_timestamp + self.fzs_interval > now: + return False + self.fzs_count -= 1 + if self.fzs_count < 1: + self.fzs_count = random.randint(0, self.fzs_range) + self.fzs_timestamp = now + return True + return False + # Check modified content.json files from peers and add modified files to bad_files # Return: Successfully queried peers [Peer, Peer...] def checkModifications(self, since=None): s = time.time() peers_try = [] # Try these peers queried = [] # Successfully queried from these peers - limit = 5 + peer_limit = 10 + updater_limit = 3 + need_queries = 3 # Wait for peers if not self.peers: @@ -588,34 +630,37 @@ class Site(object): if self.peers: break - peers_try = self.getConnectedPeers() - peers_connected_num = len(peers_try) - if peers_connected_num < limit * 2: # Add more, non-connected peers if necessary - peers_try += self.getRecentPeers(limit * 5) + if since is None: + if self.getForcedZeroSince(): + since = 0 + else: + margin = 60 * 60 * 24 + since = self.settings.get("modified", margin) - margin - if since is None: # No since defined, download from last modification time-1day - since = self.settings.get("modified", 60 * 60 * 24) - 60 * 60 * 24 + if since == 0: + peer_limit *= 4 + need_queries *= 4 - if config.verbose: - self.log.debug( - "CheckModifications: Try to get listModifications from peers: %s, connected: %s, since: %s" % - (peers_try, peers_connected_num, since) - ) + peers_try = self.getConnectedPeers() + self.getConnectablePeers(peer_limit) + + self.log.debug( + "CheckModifications: Trying to get listModifications from %s peers, %s connected, since: %s" % + (len(peers_try), len(self.getConnectedPeers()), since) + ) updaters = [] - for i in range(3): - updaters.append(gevent.spawn(self.updater, peers_try, queried, since)) + for i in range(updater_limit): + updaters.append(gevent.spawn(self.updater, peers_try, queried, need_queries, since)) - gevent.joinall(updaters, timeout=10) # Wait 10 sec to workers done query modifications + for r in range(10): + gevent.joinall(updaters, timeout=5+r) + if len(queried) >= need_queries or len(peers_try) == 0: + break + self.log.debug("CheckModifications: Waiting... (%s) succesfully queried: %s, left: %s" % + (r + 1, len(queried), len(peers_try))) - if not queried: # Start another 3 thread if first 3 is stuck - peers_try[0:0] = [peer for peer in self.getConnectedPeers() if peer.connection.connected] # Add connected peers - for _ in range(10): - gevent.joinall(updaters, timeout=10) # Wait another 10 sec if none of updaters finished - if queried: - break - - self.log.debug("CheckModifications: Queried listModifications from: %s in %.3fs since %s" % (queried, time.time() - s, since)) + self.log.debug("CheckModifications: Queried listModifications from %s peers in %.3fs since %s" % ( + len(queried), time.time() - s, since)) time.sleep(0.1) return queried From 8474abc9673f5e9c8dac850f67e71a91d85d737e Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 31 Mar 2021 09:39:22 +0700 Subject: [PATCH 060/114] Fix building the docker image --- Dockerfile | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7839cfa0..1274c7fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,42 @@ FROM alpine:3.11 -#Base settings +# Base settings ENV HOME /root +# Install packages + +RUN apk --update --no-cache --no-progress add \ + python3 python3-dev \ + libffi-dev musl-dev \ + gcc g++ make \ + automake autoconf libtool \ + openssl \ + tor + COPY requirements.txt /root/requirements.txt -#Install ZeroNet -RUN apk --update --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \ - && pip3 install -r /root/requirements.txt \ +RUN pip3 install -r /root/requirements.txt \ && apk del python3-dev gcc libffi-dev musl-dev make \ && echo "ControlPort 9051" >> /etc/tor/torrc \ && echo "CookieAuthentication 1" >> /etc/tor/torrc - + RUN python3 -V \ && python3 -m pip list \ && tor --version \ && openssl version -#Add Zeronet source +# Add Zeronet source + COPY . /root VOLUME /root/data -#Control if Tor proxy is started +# Control if Tor proxy is started ENV ENABLE_TOR false WORKDIR /root -#Set upstart command +# Set upstart command CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 -#Expose ports +# Expose ports EXPOSE 43110 26552 From adaeedf4d8dfbd2e946ca1a8f336b4d2904860d6 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 31 Mar 2021 10:51:17 +0700 Subject: [PATCH 061/114] Dockerfile: move to alpine:3.13 in order to use newer tor version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1274c7fc..bc834293 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.11 +FROM alpine:3.13 # Base settings ENV HOME /root @@ -6,7 +6,7 @@ ENV HOME /root # Install packages RUN apk --update --no-cache --no-progress add \ - python3 python3-dev \ + python3 python3-dev py3-pip \ libffi-dev musl-dev \ gcc g++ make \ automake autoconf libtool \ From 5d6fe6a63185392e464018cef5a94e82fca0beea Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 31 Mar 2021 11:16:06 +0700 Subject: [PATCH 062/114] Fix a bug introduced in c84b413f58bfc810f1e56b7e6414367f81efadf6 --- src/Connection/ConnectionServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 20bd165a..ad834c54 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -193,7 +193,7 @@ class ConnectionServer(object): connection.close("Close all connections") def handleIncomingConnection(self, sock, addr): - if self.allowsAcceptingConnections(): + if not self.allowsAcceptingConnections(): sock.close() return False From c772592c4af2da32fdacb81106c24d6e46806095 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 31 Mar 2021 11:20:37 +0700 Subject: [PATCH 063/114] New default ZeroHello address --- README-ru.md | 2 +- README-zh-cn.md | 4 ++-- README.md | 4 ++-- src/Config.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README-ru.md b/README-ru.md index 75abbfab..1b1c660b 100644 --- a/README-ru.md +++ b/README-ru.md @@ -34,7 +34,7 @@ * После запуска `zeronet.py` вы сможете посетить зайты (zeronet сайты) используя адрес `http://127.0.0.1:43110/{zeronet_address}` -(например. `http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D`). +(например. `http://127.0.0.1:43110/1HeLLoPVbqF3UEj8aWXErwTxrwkyjwGtZN`). * Когда вы посещаете новый сайт zeronet, он пытается найти пиров с помощью BitTorrent чтобы загрузить файлы сайтов (html, css, js ...) из них. * Каждый посещенный зайт также обслуживается вами. (Т.е хранится у вас на компьютере) diff --git a/README-zh-cn.md b/README-zh-cn.md index fabdb0e5..a008e154 100644 --- a/README-zh-cn.md +++ b/README-zh-cn.md @@ -33,7 +33,7 @@ * 在运行 `zeronet.py` 后，您将可以通过 `http://127.0.0.1:43110/{zeronet_address}`（例如： - `http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D`）访问 zeronet 中的站点 + `http://127.0.0.1:43110/1HeLLoPVbqF3UEj8aWXErwTxrwkyjwGtZN`）访问 zeronet 中的站点 * 在您浏览 zeronet 站点时，客户端会尝试通过 BitTorrent 网络来寻找可用的节点，从而下载需要的文件（html，css，js...） * 您将会储存每一个浏览过的站点 * 每个站点都包含一个名为 `content.json` 的文件，它储存了其他所有文件的 sha512 散列值以及一个通过站点私钥生成的签名 @@ -99,7 +99,7 @@ ## 如何创建一个 ZeroNet 站点？ - * 点击 [ZeroHello](http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D) 站点的 **⋮** > **「新建空站点」** 菜单项 + * 点击 [ZeroHello](http://127.0.0.1:43110/1HeLLoPVbqF3UEj8aWXErwTxrwkyjwGtZN) 站点的 **⋮** > **「新建空站点」** 菜单项 * 您将被**重定向**到一个全新的站点，该站点只能由您修改 * 您可以在 **data/[您的站点地址]** 目录中找到并修改网站的内容 * 修改后打开您的网站，将右上角的「0」按钮拖到左侧，然后点击底部的**签名**并**发布**按钮 diff --git a/README.md b/README.md index 5b02a2e9..1b08c868 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Decentralized websites using Bitcoin crypto and the BitTorrent network - https:/ * After starting `zeronet.py` you will be able to visit zeronet sites using `http://127.0.0.1:43110/{zeronet_address}` (eg. - `http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D`). + `http://127.0.0.1:43110/1HeLLoPVbqF3UEj8aWXErwTxrwkyjwGtZN`). * When you visit a new zeronet site, it tries to find peers using the BitTorrent network so it can download the site files (html, css, js...) from them. * Each visited site is also served by you. @@ -114,7 +114,7 @@ There is an official image, built from source at: https://hub.docker.com/r/nofis ## How can I create a ZeroNet site? - * Click on **⋮** > **"Create new, empty site"** menu item on the site [ZeroHello](http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D). + * Click on **⋮** > **"Create new, empty site"** menu item on the site [ZeroHello](http://127.0.0.1:43110/1HeLLoPVbqF3UEj8aWXErwTxrwkyjwGtZN). * You will be **redirected** to a completely new site that is only modifiable by you! * You can find and modify your site's content in **data/[yoursiteaddress]** directory * After the modifications open your site, drag the topright "0" button to left, then press **sign** and **publish** buttons on the bottom diff --git a/src/Config.py b/src/Config.py index a3d66395..fcba371e 100644 --- a/src/Config.py +++ b/src/Config.py @@ -245,7 +245,7 @@ class Config(object): self.parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name') - self.parser.add_argument('--homepage', help='Web interface Homepage', default='1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D', + self.parser.add_argument('--homepage', help='Web interface Homepage', default='1HeLLoPVbqF3UEj8aWXErwTxrwkyjwGtZN', metavar='address') self.parser.add_argument('--updatesite', help='Source code update site', default='1uPDaT3uSyWAPdCv1WkMb5hBQjWSNNACf', metavar='address') From be65ff2c40724d7936333eac12d6964ae683d35b Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 31 Mar 2021 11:27:19 +0700 Subject: [PATCH 064/114] Make more efforts of looking for peers when publishing a site --- src/Site/Site.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index 0a8be32d..1d396b08 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -865,29 +865,46 @@ class Site(object): background_publisher.finalize() del self.background_publishers[inner_path] + def waitForPeers(self, num_peers, num_connected_peers, time_limit): + start_time = time.time() + for _ in range(time_limit): + if len(self.peers) >= num_peers and len(self.getConnectedPeers()) >= num_connected_peers: + return True + self.announce(mode="more", force=True) + for wait in range(10): + self.needConnections(num=num_connected_peers) + time.sleep(2) + if len(self.peers) >= num_peers and len(self.getConnectedPeers()) >= num_connected_peers: + return True + if time.time() - start_time > time_limit: + return True + + return False + # Update content.json on peers @util.Noparallel() def publish(self, limit="default", inner_path="content.json", diffs={}, cb_progress=None): published = [] # Successfully published (Peer) publishers = [] # Publisher threads - if not self.peers: - self.announce(mode="more") - if limit == "default": limit = 5 threads = limit + self.waitForPeers(limit, limit / 2, 10) + self.waitForPeers(1, 1, 60) + peers = self.getConnectedPeers() num_connected_peers = len(peers) random.shuffle(peers) - peers = sorted(peers, key=lambda peer: peer.connection.handshake.get("rev", 0) < config.rev - 100) # Prefer newer clients + # Prefer newer clients + peers = sorted(peers, key=lambda peer: peer.connection.handshake.get("rev", 0) < config.rev - 100) - if len(peers) < limit * 2 and len(self.peers) > len(peers): # Add more, non-connected peers if necessary + # Add more, non-connected peers if necessary + if len(peers) < limit * 2 and len(self.peers) > len(peers): peers += self.getRecentPeers(limit * 2) - - peers = set(peers) + peers = set(peers) self.log.info("Publishing %s to %s/%s peers (connected: %s) diffs: %s (%.2fk)..." % ( inner_path, limit, len(self.peers), num_connected_peers, list(diffs.keys()), float(len(str(diffs))) / 1024 From abbd2c51f0c568f63e2fd8a4a87ac744d1e4d913 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 31 Mar 2021 12:55:45 +0700 Subject: [PATCH 065/114] Update ChangeLog-0.8.0.md --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md index f87f1cb9..f2d10b4a 100644 --- a/ZNE-ChangeLog/ChangeLog-0.8.0.md +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -14,13 +14,25 @@ * Peer count. More peers ==> rare announces. * Tracker count. More trackers ==> frequent announces to iterate over more trackers. * For owned zites, the activity rate doesn't drop below 0.6 to force more frequent checks. This, however, can be used to investigate which peer belongs to the zire owner. A new commnd line option `--expose_no_ownership` is introduced to disable that behavior. -* `ZeroNet` now tries harder in delivering updates to more peers in the background. +* When checking for updates, ZeroNet normally asks other peers for new data since the previous update. This however can result in losing some updates in specific conditions. To overcome this, ZeroNet now asks for the full site listing on every Nth update check. +* When asking a peer for updates, ZeroNet may see that the other peer has an older version of a file. In this case, ZeroNet sends back the notification of the new version available. The logic in 0.8.0 is generally the same, but some randomization is added which may help in distributing the "update waves" among peers. +* ZeroNet now tries harder in delivering updates to more peers in the background. +* ZeroNet also make more efforts of searching the peers before publishing updates. **Other:** * Implemented the log level overriding for separate modules for easier debugging. * Make the site block check implemented in `ContentFilter` usable from plugins and core modules via `SiteManager.isAddressBlocked()`. +## Docker Image + +* The base image upgraded from `alpine:3.11` to `alpine:3.13`. +* Tor upgraded to 0.4.4.8. + +## ZeroHello + +The default ZeroHello address changed from [1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D/](http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D/) to [1HeLLoPVbqF3UEj8aWXErwTxrwkyjwGtZN/](http://127.0.0.1:43110/1HeLLoPVbqF3UEj8aWXErwTxrwkyjwGtZN/). This is the first step in migrating away from the nofish's infrastructure which development seems to be stalled. + ## Plugins ### TrackerShare From b9ec7124f95a1c820f5698a355a42abf91a33e97 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 14:02:00 +0700 Subject: [PATCH 066/114] minor changes for debugging --- plugins/AnnounceZero/AnnounceZeroPlugin.py | 2 ++ src/Site/Site.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/AnnounceZero/AnnounceZeroPlugin.py b/plugins/AnnounceZero/AnnounceZeroPlugin.py index 623cd4b5..76ff357d 100644 --- a/plugins/AnnounceZero/AnnounceZeroPlugin.py +++ b/plugins/AnnounceZero/AnnounceZeroPlugin.py @@ -1,3 +1,4 @@ +import logging import time import itertools @@ -94,6 +95,7 @@ class SiteAnnouncerPlugin(object): tracker_ip, tracker_port = tracker_address.rsplit(":", 1) tracker_peer = Peer(str(tracker_ip), int(tracker_port), connection_server=self.site.connection_server) tracker_peer.is_tracker_connection = True + #tracker_peer.log_level = logging.INFO connection_pool[tracker_address] = tracker_peer res = tracker_peer.request("announce", request) diff --git a/src/Site/Site.py b/src/Site/Site.py index 1d396b08..010d493b 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -1357,8 +1357,13 @@ class Site(object): else: peers = list(self.peers.values()) + self.log.info("%s" % peers) + self.log.info("%s" % need_more) + + peers = peers[0:need_more * 50] + found_more = sorted( - peers[0:need_more * 50], + peers, key=lambda peer: peer.reputation, reverse=True )[0:need_more * 2] From 1c73d1a0952d6d7af8e3b563f9307fe25af6317c Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 14:26:07 +0700 Subject: [PATCH 067/114] Merge TORv3 patch From: http://127.0.0.1:43110/19HKdTAeBh5nRiKn791czY7TwRB1QNrf1Q/?:users/1HvNGwHKqhj3ZMEM53tz6jbdqe4LRpanEu:zn:dc17f896-bf3f-4962-bdd4-0a470040c9c5 Related issues: https://github.com/HelloZeroNet/ZeroNet/issues/2351 https://github.com/HelloZeroNet/ZeroNet/issues/1292 --- plugins/AnnounceZero/AnnounceZeroPlugin.py | 1 + src/Crypt/CryptEd25519.py | 340 +++++++++++++++++++++ src/Crypt/CryptRsa.py | 27 ++ src/Tor/TorManager.py | 5 +- 4 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 src/Crypt/CryptEd25519.py diff --git a/plugins/AnnounceZero/AnnounceZeroPlugin.py b/plugins/AnnounceZero/AnnounceZeroPlugin.py index 76ff357d..e71e221e 100644 --- a/plugins/AnnounceZero/AnnounceZeroPlugin.py +++ b/plugins/AnnounceZero/AnnounceZeroPlugin.py @@ -5,6 +5,7 @@ import itertools from Plugin import PluginManager from util import helper from Crypt import CryptRsa +from Crypt import CryptEd25519 allow_reload = False # No source reload supported in this plugin time_full_announced = {} # Tracker address: Last announced all site to tracker diff --git a/src/Crypt/CryptEd25519.py b/src/Crypt/CryptEd25519.py new file mode 100644 index 00000000..fc05a932 --- /dev/null +++ b/src/Crypt/CryptEd25519.py @@ -0,0 +1,340 @@ +## ZeroNet onion V3 support +## The following copied code is copied from stem.util.ed25519 official Tor Project python3 lib +## url : https://gitweb.torproject.org/stem.git/tree/stem/util/ed25519.py +## the ##modified tag means that the function has been modified respect to the one used by stem lib +## the ##custom tag means that the function has been added by me and it's not present on the stem ed25519.py file +## every comment i make begins with ## +## +# The following is copied from... +# +# https://github.com/pyca/ed25519 +# +# This is under the CC0 license. For more information please see... +# +# https://github.com/pyca/cryptography/issues/5068 + +# ed25519.py - Optimized version of the reference implementation of Ed25519 +# +# Written in 2011? by Daniel J. Bernstein +# 2013 by Donald Stufft +# 2013 by Alex Gaynor +# 2013 by Greg Price +# +# To the extent possible under law, the author(s) have dedicated all copyright +# and related and neighboring rights to this software to the public domain +# worldwide. This software is distributed without any warranty. +# +# You should have received a copy of the CC0 Public Domain Dedication along +# with this software. If not, see +# . + +""" +NB: This code is not safe for use with secret keys or secret data. +The only safe use of this code is for verifying signatures on public messages. + +Functions for computing the public key of a secret key and for signing +a message are included, namely publickey_unsafe and signature_unsafe, +for testing purposes only. + +The root of the problem is that Python's long-integer arithmetic is +not designed for use in cryptography. Specifically, it may take more +or less time to execute an operation depending on the values of the +inputs, and its memory access patterns may also depend on the inputs. +This opens it to timing and cache side-channel attacks which can +disclose data to an attacker. We rely on Python's long-integer +arithmetic, so we cannot handle secrets without risking their disclosure. +""" + +import hashlib +import operator +import sys +import base64 + + +__version__ = "1.0.dev0" + + +# Useful for very coarse version differentiation. +PY3 = sys.version_info[0] == 3 + +if PY3: + indexbytes = operator.getitem + intlist2bytes = bytes + int2byte = operator.methodcaller("to_bytes", 1, "big") +else: + int2byte = chr + range = xrange + + def indexbytes(buf, i): + return ord(buf[i]) + + def intlist2bytes(l): + return b"".join(chr(c) for c in l) + + +b = 256 +q = 2 ** 255 - 19 +l = 2 ** 252 + 27742317777372353535851937790883648493 + + +def H(m): + return hashlib.sha512(m).digest() + + +def pow2(x, p): + """== pow(x, 2**p, q)""" + while p > 0: + x = x * x % q + p -= 1 + return x + + +def inv(z): + """$= z^{-1} \mod q$, for z != 0""" + # Adapted from curve25519_athlon.c in djb's Curve25519. + z2 = z * z % q # 2 + z9 = pow2(z2, 2) * z % q # 9 + z11 = z9 * z2 % q # 11 + z2_5_0 = (z11 * z11) % q * z9 % q # 31 == 2^5 - 2^0 + z2_10_0 = pow2(z2_5_0, 5) * z2_5_0 % q # 2^10 - 2^0 + z2_20_0 = pow2(z2_10_0, 10) * z2_10_0 % q # ... + z2_40_0 = pow2(z2_20_0, 20) * z2_20_0 % q + z2_50_0 = pow2(z2_40_0, 10) * z2_10_0 % q + z2_100_0 = pow2(z2_50_0, 50) * z2_50_0 % q + z2_200_0 = pow2(z2_100_0, 100) * z2_100_0 % q + z2_250_0 = pow2(z2_200_0, 50) * z2_50_0 % q # 2^250 - 2^0 + return pow2(z2_250_0, 5) * z11 % q # 2^255 - 2^5 + 11 = q - 2 + + +d = -121665 * inv(121666) % q +I = pow(2, (q - 1) // 4, q) + + +def xrecover(y): + xx = (y * y - 1) * inv(d * y * y + 1) + x = pow(xx, (q + 3) // 8, q) + + if (x * x - xx) % q != 0: + x = (x * I) % q + + if x % 2 != 0: + x = q-x + + return x + + +By = 4 * inv(5) +Bx = xrecover(By) +B = (Bx % q, By % q, 1, (Bx * By) % q) +ident = (0, 1, 1, 0) + + +def edwards_add(P, Q): + # This is formula sequence 'addition-add-2008-hwcd-3' from + # http://www.hyperelliptic.org/EFD/g1p/auto-twisted-extended-1.html + (x1, y1, z1, t1) = P + (x2, y2, z2, t2) = Q + + a = (y1-x1)*(y2-x2) % q + b = (y1+x1)*(y2+x2) % q + c = t1*2*d*t2 % q + dd = z1*2*z2 % q + e = b - a + f = dd - c + g = dd + c + h = b + a + x3 = e*f + y3 = g*h + t3 = e*h + z3 = f*g + + return (x3 % q, y3 % q, z3 % q, t3 % q) + + +def edwards_double(P): + # This is formula sequence 'dbl-2008-hwcd' from + # http://www.hyperelliptic.org/EFD/g1p/auto-twisted-extended-1.html + (x1, y1, z1, t1) = P + + a = x1*x1 % q + b = y1*y1 % q + c = 2*z1*z1 % q + # dd = -a + e = ((x1+y1)*(x1+y1) - a - b) % q + g = -a + b # dd + b + f = g - c + h = -a - b # dd - b + x3 = e*f + y3 = g*h + t3 = e*h + z3 = f*g + + return (x3 % q, y3 % q, z3 % q, t3 % q) + + +def scalarmult(P, e): + if e == 0: + return ident + Q = scalarmult(P, e // 2) + Q = edwards_double(Q) + if e & 1: + Q = edwards_add(Q, P) + return Q + + +# Bpow[i] == scalarmult(B, 2**i) +Bpow = [] + + +def make_Bpow(): + P = B + for i in range(253): + Bpow.append(P) + P = edwards_double(P) +make_Bpow() + + +def scalarmult_B(e): + """ + Implements scalarmult(B, e) more efficiently. + """ + # scalarmult(B, l) is the identity + e = e % l + P = ident + for i in range(253): + if e & 1: + P = edwards_add(P, Bpow[i]) + e = e // 2 + assert e == 0, e + return P + + +def encodeint(y): + bits = [(y >> i) & 1 for i in range(b)] + return b''.join([ + int2byte(sum([bits[i * 8 + j] << j for j in range(8)])) + for i in range(b//8) + ]) + + +def encodepoint(P): + (x, y, z, t) = P + zi = inv(z) + x = (x * zi) % q + y = (y * zi) % q + bits = [(y >> i) & 1 for i in range(b - 1)] + [x & 1] + return b''.join([ + int2byte(sum([bits[i * 8 + j] << j for j in range(8)])) + for i in range(b // 8) + ]) + + +def bit(h, i): + return (indexbytes(h, i // 8) >> (i % 8)) & 1 + +##modified +def publickey_unsafe(sk): + """ + Not safe to use with secret keys or secret data. + + See module docstring. This function should be used for testing only. + """ + ##h = H(sk) + h = sk + a = 2 ** (b - 2) + sum(2 ** i * bit(h, i) for i in range(3, b - 2)) + A = scalarmult_B(a) + return encodepoint(A) + +##custom +## from stem.util.str_tools._to_unicode_impl +## from https://gitweb.torproject.org/stem.git/tree/stem/util/str_tools.py#n80 +def to_unicode_impl(msg): + if msg is not None and not isinstance(msg, str): + return msg.decode('utf-8', 'replace') + else: + return msg + +##custom +## rewritten stem.descriptor.hidden_service.address_from_identity_key +## from https://gitweb.torproject.org/stem.git/tree/stem/descriptor/hidden_service.py#n1088 +def publickey_to_onionaddress(key): + CHECKSUM_CONSTANT = b'.onion checksum' + ## version = stem.client.datatype.Size.CHAR.pack(3) + version = b'\x03' + checksum = hashlib.sha3_256(CHECKSUM_CONSTANT + key + version).digest()[:2] + onion_address = base64.b32encode(key + checksum + version) + return to_unicode_impl(onion_address + b'.onion').lower() + + +def Hint(m): + h = H(m) + return sum(2 ** i * bit(h, i) for i in range(2 * b)) + +##modified +def signature_unsafe(m, sk, pk): + """ + Not safe to use with secret keys or secret data. + + See module docstring. This function should be used for testing only. + """ + ##h = H(sk) + h = sk + a = 2 ** (b - 2) + sum(2 ** i * bit(h, i) for i in range(3, b - 2)) + r = Hint( + intlist2bytes([indexbytes(h, j) for j in range(b // 8, b // 4)]) + m + ) + R = scalarmult_B(r) + S = (r + Hint(encodepoint(R) + pk + m) * a) % l + return encodepoint(R) + encodeint(S) + + +def isoncurve(P): + (x, y, z, t) = P + return (z % q != 0 and + x*y % q == z*t % q and + (y*y - x*x - z*z - d*t*t) % q == 0) + + +def decodeint(s): + return sum(2 ** i * bit(s, i) for i in range(0, b)) + + +def decodepoint(s): + y = sum(2 ** i * bit(s, i) for i in range(0, b - 1)) + x = xrecover(y) + if x & 1 != bit(s, b-1): + x = q - x + P = (x, y, 1, (x*y) % q) + if not isoncurve(P): + raise ValueError("decoding point that is not on curve") + return P + + +class SignatureMismatch(Exception): + pass + + +def checkvalid(s, m, pk): + """ + Not safe to use when any argument is secret. + + See module docstring. This function should be used only for + verifying public signatures of public messages. + """ + if len(s) != b // 4: + raise ValueError("signature length is wrong") + + if len(pk) != b // 8: + raise ValueError("public-key length is wrong") + + R = decodepoint(s[:b // 8]) + A = decodepoint(pk) + S = decodeint(s[b // 8:b // 4]) + h = Hint(encodepoint(R) + pk + m) + + (x1, y1, z1, t1) = P = scalarmult_B(S) + (x2, y2, z2, t2) = Q = edwards_add(R, scalarmult(A, h)) + + if (not isoncurve(P) or not isoncurve(Q) or + (x1*z2 - x2*z1) % q != 0 or (y1*z2 - y2*z1) % q != 0): + raise SignatureMismatch("signature does not pass verification") diff --git a/src/Crypt/CryptRsa.py b/src/Crypt/CryptRsa.py index 494c4d24..02df8f41 100644 --- a/src/Crypt/CryptRsa.py +++ b/src/Crypt/CryptRsa.py @@ -4,6 +4,13 @@ import hashlib def sign(data, privatekey): import rsa from rsa import pkcs1 + from Crypt import CryptEd25519 + ## v3 = 88 + if len(privatekey) == 88: + prv_key = base64.b64decode(privatekey) + pub_key = CryptEd25519.publickey_unsafe(prv_key) + sign = CryptEd25519.signature_unsafe(data, prv_key, pub_key) + return sign if "BEGIN RSA PRIVATE KEY" not in privatekey: privatekey = "-----BEGIN RSA PRIVATE KEY-----\n%s\n-----END RSA PRIVATE KEY-----" % privatekey @@ -15,6 +22,16 @@ def sign(data, privatekey): def verify(data, publickey, sign): import rsa from rsa import pkcs1 + from Crypt import CryptEd25519 + + if len(publickey) == 32: + try: + valid = CryptEd25519.checkvalid(sign, data, publickey) + valid = 'SHA-256' + except Exception as err: + print(err) + valid = False + return valid pub = rsa.PublicKey.load_pkcs1(publickey, format="DER") try: @@ -24,9 +41,15 @@ def verify(data, publickey, sign): return valid def privatekeyToPublickey(privatekey): + from Crypt import CryptEd25519 import rsa from rsa import pkcs1 + if len(privatekey) == 88: + prv_key = base64.b64decode(privatekey) + pub_key = CryptEd25519.publickey_unsafe(prv_key) + return pub_key + if "BEGIN RSA PRIVATE KEY" not in privatekey: privatekey = "-----BEGIN RSA PRIVATE KEY-----\n%s\n-----END RSA PRIVATE KEY-----" % privatekey @@ -35,4 +58,8 @@ def privatekeyToPublickey(privatekey): return pub.save_pkcs1("DER") def publickeyToOnion(publickey): + from Crypt import CryptEd25519 + if len(publickey) == 32: + addr = CryptEd25519.publickey_to_onionaddress(publickey)[:-6] + return addr return base64.b32encode(hashlib.sha1(publickey).digest()[:10]).lower().decode("ascii") diff --git a/src/Tor/TorManager.py b/src/Tor/TorManager.py index 7e5c8bb0..8de33b4b 100644 --- a/src/Tor/TorManager.py +++ b/src/Tor/TorManager.py @@ -13,6 +13,7 @@ import gevent from Config import config from Crypt import CryptRsa +from Crypt import CryptEd25519 from Site import SiteManager import socks from gevent.lock import RLock @@ -214,8 +215,8 @@ class TorManager(object): return False def makeOnionAndKey(self): - res = self.request("ADD_ONION NEW:RSA1024 port=%s" % self.fileserver_port) - match = re.search("ServiceID=([A-Za-z0-9]+).*PrivateKey=RSA1024:(.*?)[\r\n]", res, re.DOTALL) + res = self.request("ADD_ONION NEW:ED25519-V3 port=%s" % self.fileserver_port) + match = re.search("ServiceID=([A-Za-z0-9]+).*PrivateKey=ED25519-V3:(.*?)[\r\n]", res, re.DOTALL) if match: onion_address, onion_privatekey = match.groups() return (onion_address, onion_privatekey) From 72e5d3df640a08edd2913d7f9423ea9f57613171 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 14:28:22 +0700 Subject: [PATCH 068/114] Fix typo in "All server stopped" --- src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index 209bb9d2..ec04c9e3 100644 --- a/src/main.py +++ b/src/main.py @@ -155,7 +155,7 @@ class Actions(object): logging.info("Starting servers....") gevent.joinall([gevent.spawn(ui_server.start), gevent.spawn(file_server.start)]) - logging.info("All server stopped") + logging.info("All servers stopped") # Site commands From 164f5199a9e4f107b0fbed1f843cb908949755ca Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 16:28:28 +0700 Subject: [PATCH 069/114] Add default onion v3 tracker addresses --- src/Config.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Config.py b/src/Config.py index fcba371e..67f65f7b 100644 --- a/src/Config.py +++ b/src/Config.py @@ -80,6 +80,17 @@ class Config(object): # Create command line arguments def createArguments(self): trackers = [ + # by zeroseed at http://127.0.0.1:43110/19HKdTAeBh5nRiKn791czY7TwRB1QNrf1Q/?:users/1HvNGwHKqhj3ZMEM53tz6jbdqe4LRpanEu:zn:dc17f896-bf3f-4962-bdd4-0a470040c9c5 + "zero://k5w77dozo3hy5zualyhni6vrh73iwfkaofa64abbilwyhhd3wgenbjqd.onion:15441", + "zero://2kcb2fqesyaevc4lntogupa4mkdssth2ypfwczd2ov5a3zo6ytwwbayd.onion:15441", + "zero://my562dxpjropcd5hy3nd5pemsc4aavbiptci5amwxzbelmzgkkuxpvid.onion:15441", + "zero://pn4q2zzt2pw4nk7yidxvsxmydko7dfibuzxdswi6gu6ninjpofvqs2id.onion:15441", + "zero://6i54dd5th73oelv636ivix6sjnwfgk2qsltnyvswagwphub375t3xcad.onion:15441", + "zero://tl74auz4tyqv4bieeclmyoe4uwtoc2dj7fdqv4nc4gl5j2bwg2r26bqd.onion:15441", + "zero://wlxav3szbrdhest4j7dib2vgbrd7uj7u7rnuzg22cxbih7yxyg2hsmid.onion:15441", + "zero://zy7wttvjtsijt5uwmlar4yguvjc2gppzbdj4v6bujng6xwjmkdg7uvqd.onion:15441", + + # ZeroNet 0.7.2 defaults: "zero://boot3rdez4rzn36x.onion:15441", "zero://zero.booth.moe#f36ca555bee6ba216b14d10f38c16f7769ff064e0e37d887603548cc2e64191d:443", # US/NY "udp://tracker.coppersurfer.tk:6969", # DE From d5652eaa511e576c02dba2097c3e6506bae66f36 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 16:42:21 +0700 Subject: [PATCH 070/114] Update ChangeLog-0.8.0.md --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md index f2d10b4a..17266268 100644 --- a/ZNE-ChangeLog/ChangeLog-0.8.0.md +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -3,6 +3,8 @@ **Network:** +* Added support of Onion v3 addresses. Thanks to @anonymoose and @zeroseed. +* Added a few Onion v3 tracker addresses. * Reworked the algorithm of checking zite updates on startup / after the network outages / periodically. ZeroNet tries not to spam too many update queries at once in order to prevent network overload. (Which especially the issue when running over Tor.) At the same time, it tries to keep balance between frequent checks for frequently updating zites and ensuring that all the zites are checked in some reasonable time interval. Tests show that the full check cycle for a peer that hosts 800+ zites and is connected over Tor can take up to several hours. We cannot significantly reduce this time, since the Tor throughput is the bottleneck. Running more checks at once just results in more connections to fail. The other bottleneck is the HDD throughput. Increasing the parallelization doesn't help in this case as well. So the implemented solution **decreases** the concurency. * Improved the Internet outage detection and the recovery procedures after the Internet be back. ZeroNet "steps back" and schedules rechecking zites that were checked shortly before the Internet connection get lost. The network outage detection normally has some lag, so the recently checked zites are better to checked again. * When the network is down, reduce the frequency of connection attempts to prevent overloading Tor with hanged connections. From d2b65c550c405237087b09ba2f84477fbac3f042 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 16:43:13 +0700 Subject: [PATCH 071/114] Minor fix in ChangeLog-0.8.0.md --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md index 17266268..f662d353 100644 --- a/ZNE-ChangeLog/ChangeLog-0.8.0.md +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -7,7 +7,7 @@ * Added a few Onion v3 tracker addresses. * Reworked the algorithm of checking zite updates on startup / after the network outages / periodically. ZeroNet tries not to spam too many update queries at once in order to prevent network overload. (Which especially the issue when running over Tor.) At the same time, it tries to keep balance between frequent checks for frequently updating zites and ensuring that all the zites are checked in some reasonable time interval. Tests show that the full check cycle for a peer that hosts 800+ zites and is connected over Tor can take up to several hours. We cannot significantly reduce this time, since the Tor throughput is the bottleneck. Running more checks at once just results in more connections to fail. The other bottleneck is the HDD throughput. Increasing the parallelization doesn't help in this case as well. So the implemented solution **decreases** the concurency. * Improved the Internet outage detection and the recovery procedures after the Internet be back. ZeroNet "steps back" and schedules rechecking zites that were checked shortly before the Internet connection get lost. The network outage detection normally has some lag, so the recently checked zites are better to checked again. -* When the network is down, reduce the frequency of connection attempts to prevent overloading Tor with hanged connections. +* When the network is down, reduce the frequency of connection attempts to prevent overloading Tor with hanged connections. * The connection handling code had several bugs that were hidden by silently ignored exceptions. These were fixed, but some new ones might be introduced. * For each zite the activity rate is calculated based on the last modification time. The milestone values are 1 hour, 5 hours, 24 hours, 3 days and 7 days. The activity rate is used to scale frequency of various maintenance routines, including update checks, reannounces, dead connection checks etc. * The activity rate is also used to calculate the minimum preferred number of active connections per each zite. From 769a2c08ddb86066097c21f8ce53bf57b007a6cf Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 17:25:47 +0700 Subject: [PATCH 072/114] Fix possible infinite growing of the `SafeRe` regexp cache by limiting the cache size --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 1 + src/util/SafeRe.py | 42 +++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md index f662d353..9990c683 100644 --- a/ZNE-ChangeLog/ChangeLog-0.8.0.md +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -25,6 +25,7 @@ * Implemented the log level overriding for separate modules for easier debugging. * Make the site block check implemented in `ContentFilter` usable from plugins and core modules via `SiteManager.isAddressBlocked()`. +* Fixed possible infinite growing of the `SafeRe` regexp cache by limiting the cache size. ## Docker Image diff --git a/src/util/SafeRe.py b/src/util/SafeRe.py index 6018e2d3..2d519a79 100644 --- a/src/util/SafeRe.py +++ b/src/util/SafeRe.py @@ -1,10 +1,16 @@ import re +import logging + +log = logging.getLogger("SafeRe") + class UnsafePatternError(Exception): pass +max_cache_size = 1000 cached_patterns = {} +old_cached_patterns = {} def isSafePattern(pattern): @@ -22,11 +28,35 @@ def isSafePattern(pattern): return True -def match(pattern, *args, **kwargs): +def compilePattern(pattern): + global cached_patterns + global old_cached_patterns + cached_pattern = cached_patterns.get(pattern) if cached_pattern: - return cached_pattern.match(*args, **kwargs) - else: - if isSafePattern(pattern): - cached_patterns[pattern] = re.compile(pattern) - return cached_patterns[pattern].match(*args, **kwargs) + return cached_pattern + + cached_pattern = old_cached_patterns.get(pattern) + if cached_pattern: + del old_cached_patterns[pattern] + cached_patterns[pattern] = cached_pattern + return cached_pattern + + if isSafePattern(pattern): + cached_pattern = re.compile(pattern) + cached_patterns[pattern] = cached_pattern + log.debug("Compiled new pattern: %s" % pattern) + log.debug("Cache size: %d + %d" % (len(cached_patterns), len(old_cached_patterns))) + + if len(cached_patterns) > max_cache_size: + old_cached_patterns = cached_patterns + cached_patterns = {} + log.debug("Size limit reached. Rotating cache.") + log.debug("Cache size: %d + %d" % (len(cached_patterns), len(old_cached_patterns))) + + return cached_pattern + + +def match(pattern, *args, **kwargs): + cached_pattern = compilePattern(pattern) + return cached_pattern.match(*args, **kwargs) From 488fd4045ee79489d21fbb31bf5ad12645ebb017 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 17:28:03 +0700 Subject: [PATCH 073/114] Update ChangeLog-0.8.0.md --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md index 9990c683..2b532d62 100644 --- a/ZNE-ChangeLog/ChangeLog-0.8.0.md +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -36,6 +36,9 @@ The default ZeroHello address changed from [1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D/](http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D/) to [1HeLLoPVbqF3UEj8aWXErwTxrwkyjwGtZN/](http://127.0.0.1:43110/1HeLLoPVbqF3UEj8aWXErwTxrwkyjwGtZN/). This is the first step in migrating away from the nofish's infrastructure which development seems to be stalled. +* Make #Dashboard .tracker-status wider to fit onion v3 addresses. +* Added link to /Stats to the menu. + ## Plugins ### TrackerShare From 23ef37374b6b641a2ea342a93cb1572811428eb5 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 17:39:39 +0700 Subject: [PATCH 074/114] Various fixes in ChangeLog-0.8.0.md --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md index 2b532d62..a2bba51b 100644 --- a/ZNE-ChangeLog/ChangeLog-0.8.0.md +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -5,8 +5,8 @@ * Added support of Onion v3 addresses. Thanks to @anonymoose and @zeroseed. * Added a few Onion v3 tracker addresses. -* Reworked the algorithm of checking zite updates on startup / after the network outages / periodically. ZeroNet tries not to spam too many update queries at once in order to prevent network overload. (Which especially the issue when running over Tor.) At the same time, it tries to keep balance between frequent checks for frequently updating zites and ensuring that all the zites are checked in some reasonable time interval. Tests show that the full check cycle for a peer that hosts 800+ zites and is connected over Tor can take up to several hours. We cannot significantly reduce this time, since the Tor throughput is the bottleneck. Running more checks at once just results in more connections to fail. The other bottleneck is the HDD throughput. Increasing the parallelization doesn't help in this case as well. So the implemented solution **decreases** the concurency. -* Improved the Internet outage detection and the recovery procedures after the Internet be back. ZeroNet "steps back" and schedules rechecking zites that were checked shortly before the Internet connection get lost. The network outage detection normally has some lag, so the recently checked zites are better to checked again. +* Reworked the algorithm of checking zite updates on startup / after the network outages / periodically. ZeroNet tries not to spam too many update queries at once in order to prevent network overload. (Which especially the issue when running over Tor.) At the same time, it tries to keep balance between frequent checks for frequently updating zites and ensuring that all the zites are checked in some reasonable time interval. Tests show that the full check cycle for a peer that hosts 800+ zites and is connected over Tor can take up to several hours. We cannot significantly reduce this time, since the Tor throughput is the bottleneck. Running more checks at once just results in more connections to fail. The other bottleneck is the HDD throughput. Increasing the parallelization doesn't help in this case as well. So the implemented solution do actually **decreases** the parallelization. +* Improved the Internet outage detection and the recovery procedures after the Internet be back. ZeroNet "steps back" and schedules rechecking zites that were checked shortly before the Internet connection get lost. The network outage detection normally has some lag, so the recently checked zites are better to be checked again. * When the network is down, reduce the frequency of connection attempts to prevent overloading Tor with hanged connections. * The connection handling code had several bugs that were hidden by silently ignored exceptions. These were fixed, but some new ones might be introduced. * For each zite the activity rate is calculated based on the last modification time. The milestone values are 1 hour, 5 hours, 24 hours, 3 days and 7 days. The activity rate is used to scale frequency of various maintenance routines, including update checks, reannounces, dead connection checks etc. @@ -15,11 +15,11 @@ * Activity. More activity ==> frequent announces. * Peer count. More peers ==> rare announces. * Tracker count. More trackers ==> frequent announces to iterate over more trackers. -* For owned zites, the activity rate doesn't drop below 0.6 to force more frequent checks. This, however, can be used to investigate which peer belongs to the zire owner. A new commnd line option `--expose_no_ownership` is introduced to disable that behavior. +* For owned zites, the activity rate doesn't drop below 0.6 to force more frequent checks. This, however, can be used to investigate which peer belongs to the zite owner. A new commnd line option `--expose_no_ownership` is introduced to disable that behavior. * When checking for updates, ZeroNet normally asks other peers for new data since the previous update. This however can result in losing some updates in specific conditions. To overcome this, ZeroNet now asks for the full site listing on every Nth update check. * When asking a peer for updates, ZeroNet may see that the other peer has an older version of a file. In this case, ZeroNet sends back the notification of the new version available. The logic in 0.8.0 is generally the same, but some randomization is added which may help in distributing the "update waves" among peers. * ZeroNet now tries harder in delivering updates to more peers in the background. -* ZeroNet also make more efforts of searching the peers before publishing updates. +* ZeroNet also makes more efforts of searching the peers before publishing updates. **Other:** @@ -49,12 +49,12 @@ Changes in the plugin: * The default total tracker limit increased up to 20 from 5. This reduces the probability of accidental splitting the network into segments that operate with disjoint sets of trackers. * The plugin now shares any types of trackers, working both on IP and Onion networks, not limiting solely to Zero-based IP trackers. -* The plugin now takes into account not only the total number of known working trackers, but also does it on per protocol basis. The defaults are to keep 10 trackers for Zero protocol and 5 trackers per each other protocol. The available protocols are detected automatically. (For example, UDP is considered disabled, when working in the Tor-always mode.) The following protocols are recognized: `zero://` (Zero Tracker), `udp://` (UDP-based bitTorrent tracker), `http://` or `https://` (HTTP or HTTPS-based bitTorrent tracker; considered the same protocol). In case of new protocols implemented by third-party plugins, trackers are grouped automatically by the protocol prefix. -* Reworked the interaction of this plugin and the Zero tracker (`Bootstrapper`) plugin. Previously: `AnnounceShare` checks if `Bootstrapper` is running and adds its addresses to the list. Now: the tracker explicitly asks `TrackerShare` to add its addresses. -* The plugin allows adjustings the behaviour per each tracker entry with the following boolean fields: +* The plugin now takes into account not only the total number of known working trackers, but also does it on per protocol basis. The defaults are to keep 10 trackers for Zero protocol and 5 trackers per each other protocol. The available protocols are detected automatically. (For example, UDP is considered disabled, when working in the Tor-always mode.) The following protocols are recognized: `zero://` (Zero Tracker), `udp://` (UDP-based BitTorrent tracker), `http://` or `https://` (HTTP or HTTPS-based BitTorrent tracker; considered the same protocol). In case of new protocols implemented by third-party plugins, trackers are grouped automatically by the protocol prefix. +* Reworked the interaction of this plugin and the Zero tracker (`Bootstrapper`) plugin. Previously: `AnnounceShare` checks if `Bootstrapper` is running and adds its addresses to the list. Now: `Bootstrapper` explicitly asks `TrackerShare` to add its addresses. +* The plugin allows adjusting the behaviour per each tracker entry with the following boolean fields: * `my` — "My" trackers get never deleted on announce errors, but aren't saved between restarts. Designed to be used by tracker implementation plugins. `TrackerShare` acts more persistently in recommending "my" trackers to other peers. * `persistent` — "Persistent" trackers get never deleted, when unresponsive. - * `private` — "Private" trackers are never exposed to other peer in response of the getTrackers command. + * `private` — "Private" trackers are never exposed to other peers in response of the getTrackers command. ### TrackerZero @@ -74,6 +74,6 @@ Running over TOR doesn't seem to be stable so far. Any investigations and bug re ### TrackerList -`TrackerZero` is a new plugin for fetching tracker lists. By default is list is fetched from [https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt](https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt). +`TrackerList` is a new plugin for fetching tracker lists. By default is list is fetched from [https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt](https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt). TODO: add support of fetching from ZeroNet URLs From ba6295f793ea6dc1831eb72b0e75469d5e9ccd21 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 18:28:03 +0700 Subject: [PATCH 075/114] Add tests directly to SafeRe.py --- src/util/SafeRe.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/util/SafeRe.py b/src/util/SafeRe.py index 2d519a79..c9e14c0d 100644 --- a/src/util/SafeRe.py +++ b/src/util/SafeRe.py @@ -60,3 +60,34 @@ def compilePattern(pattern): def match(pattern, *args, **kwargs): cached_pattern = compilePattern(pattern) return cached_pattern.match(*args, **kwargs) + +################################################################################ + +# TESTS + +def testSafePattern(pattern): + try: + return isSafePattern(pattern) + except UnsafePatternError as err: + return False + + +# Some real examples to make sure it works as expected +assert testSafePattern('(data/mp4/.*|updater/.*)') +assert testSafePattern('((js|css)/(?!all.(js|css)))|.git') + + +# Unsafe cases: + +# ((?!json).)*$ not allowed, because of ) before the * character. Possible fix: .*(?!json)$ +assert not testSafePattern('((?!json).)*$') +assert testSafePattern('.*(?!json)$') + +# (.*.epub|.*.jpg|.*.jpeg|.*.png|data/.*.gif|.*.avi|.*.ogg|.*.webm|.*.mp4|.*.mp3|.*.mkv|.*.eot) not allowed, because it has 12 .* repetition patterns. Possible fix: .*(epub|jpg|jpeg|png|data/gif|avi|ogg|webm|mp4|mp3|mkv|eot) +assert not testSafePattern('(.*.epub|.*.jpg|.*.jpeg|.*.png|data/.*.gif|.*.avi|.*.ogg|.*.webm|.*.mp4|.*.mp3|.*.mkv|.*.eot)') +assert testSafePattern('.*(epub|jpg|jpeg|png|data/gif|avi|ogg|webm|mp4|mp3|mkv|eot)') + +# FIXME: https://github.com/HelloZeroNet/ZeroNet/issues/2757 +#assert not testSafePattern('a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + +################################################################################ From 2a25d61b968a21aa98c6db2ca9d64f1bbdc54773 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 19:01:55 +0700 Subject: [PATCH 076/114] Fix https://github.com/HelloZeroNet/ZeroNet/issues/2757 --- src/util/SafeRe.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/util/SafeRe.py b/src/util/SafeRe.py index c9e14c0d..8c394a84 100644 --- a/src/util/SafeRe.py +++ b/src/util/SafeRe.py @@ -21,9 +21,10 @@ def isSafePattern(pattern): if unsafe_pattern_match: raise UnsafePatternError("Potentially unsafe part of the pattern: %s in %s" % (unsafe_pattern_match.group(0), pattern)) - repetitions = re.findall(r"\.[\*\{\+]", pattern) - if len(repetitions) >= 10: - raise UnsafePatternError("More than 10 repetitions of %s in %s" % (repetitions[0], pattern)) + repetitions1 = re.findall(r"\.[\*\{\+]", pattern) + repetitions2 = re.findall(r"[^(][?]", pattern) + if len(repetitions1) + len(repetitions2) >= 10: + raise UnsafePatternError("More than 10 repetitions in %s" % pattern) return True @@ -87,7 +88,11 @@ assert testSafePattern('.*(?!json)$') assert not testSafePattern('(.*.epub|.*.jpg|.*.jpeg|.*.png|data/.*.gif|.*.avi|.*.ogg|.*.webm|.*.mp4|.*.mp3|.*.mkv|.*.eot)') assert testSafePattern('.*(epub|jpg|jpeg|png|data/gif|avi|ogg|webm|mp4|mp3|mkv|eot)') -# FIXME: https://github.com/HelloZeroNet/ZeroNet/issues/2757 -#assert not testSafePattern('a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') +# https://github.com/HelloZeroNet/ZeroNet/issues/2757 +assert not testSafePattern('a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') +assert not testSafePattern('a?a?a?a?a?a?a?x.{0,1}x.{0,1}x.{0,1}') +assert testSafePattern('a?a?a?a?a?a?a?x.{0,1}x.{0,1}') +assert not testSafePattern('a?a?a?a?a?a?a?x.*x.*x.*') +assert testSafePattern('a?a?a?a?a?a?a?x.*x.*') ################################################################################ From 46bea950024aa606a9e8c5f63ebadc2361533f09 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 20:43:52 +0700 Subject: [PATCH 077/114] Small cleanup in TrackerSharePlugin.py --- plugins/TrackerShare/TrackerSharePlugin.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index e6b4454b..99ebf692 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -443,6 +443,16 @@ class TrackerStorage(object): self.log.info("Discovered %s new trackers from %s/%s peers in %.3fs" % (num_trackers_discovered, num_success, len(peers), time.time() - s)) + def checkDiscoveringTrackers(self, peers): + if not peers or len(peers) < 1: + return + + now = time.time() + if self.time_discover + self.tracker_discover_time_interval >= now: + return + + self.time_discover = now + gevent.spawn(self.discoverTrackers, peers) if "tracker_storage" not in locals(): tracker_storage = TrackerStorage() @@ -452,9 +462,7 @@ if "tracker_storage" not in locals(): class SiteAnnouncerPlugin(object): def getTrackers(self): tracker_storage.setSiteAnnouncer(self) - if tracker_storage.time_discover < time.time() - tracker_storage.tracker_discover_time_interval: - tracker_storage.time_discover = time.time() - gevent.spawn(tracker_storage.discoverTrackers, self.site.getConnectedPeers()) + tracker_storage.checkDiscoveringTrackers(self.site.getConnectedPeers()) trackers = super(SiteAnnouncerPlugin, self).getTrackers() shared_trackers = list(tracker_storage.getTrackers().keys()) if shared_trackers: From 7e438a90e15cb9e034355e22a942c7055a7febcc Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 21:10:04 +0700 Subject: [PATCH 078/114] Don't spam in log.info() from getRecentPeers() --- src/Site/Site.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index 010d493b..536f3faa 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -1357,8 +1357,8 @@ class Site(object): else: peers = list(self.peers.values()) - self.log.info("%s" % peers) - self.log.info("%s" % need_more) + self.log.debug("getRecentPeers: peers = %s" % peers) + self.log.debug("getRecentPeers: need_more = %s" % need_more) peers = peers[0:need_more * 50] From 75bba6ca1a6ead87b1b4e766741cfc53dbfa276d Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 22:40:34 +0700 Subject: [PATCH 079/114] Be more verbose about starting/stopping FileServer threads --- src/File/FileServer.py | 38 +++++++++++++++++++++++++++++--------- src/Ui/UiServer.py | 2 +- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index c2a7d7d0..73c86bd4 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -322,7 +322,7 @@ class FileServer(ConnectionServer): task_nr = self.update_sites_task_next_nr self.update_sites_task_next_nr += 1 - task_description = "updateSites: #%d, check_files=%s" % (task_nr, check_files) + task_description = "updateSites [#%d, check_files=%s]" % (task_nr, check_files) log.info("%s: started", task_description) # Don't wait port opening on first startup. Do the instant check now. @@ -363,6 +363,9 @@ class FileServer(ConnectionServer): while self.running: self.waitForInternetOnline() + if self.stopping: + break + thread = self.update_pool.spawn(self.updateSite, site, check_files=check_files) if check_files: # Limit the concurency @@ -371,6 +374,9 @@ class FileServer(ConnectionServer): if not self.waitForInternetOnline(): break + if self.stopping: + break + if time.time() - progress_print_time > 60: progress_print_time = time.time() time_spent = time.time() - start_time @@ -387,9 +393,14 @@ class FileServer(ConnectionServer): time_left ) - log.info("%s: finished in %.2fs", task_description, time.time() - start_time) + if self.stopping: + log.info("%s: stopped", task_description) + else: + log.info("%s: finished in %.2fs", task_description, time.time() - start_time) def sitesMaintenanceThread(self, mode="full"): + log.info("sitesMaintenanceThread(%s) started" % mode) + startup = True short_timeout = 2 @@ -460,6 +471,7 @@ class FileServer(ConnectionServer): site_addresses = None startup = False + log.info("sitesMaintenanceThread(%s) stopped" % mode) def keepAliveThread(self): # This thread is mostly useless on a system under load, since it never does @@ -475,6 +487,7 @@ class FileServer(ConnectionServer): # are interested in connecting to them), we initiate some traffic by # performing the update for a random site. It's way better than just # silly pinging a random peer for no profit. + log.info("keepAliveThread started") while self.running: self.waitForInternetOnline() @@ -499,6 +512,7 @@ class FileServer(ConnectionServer): now - last_activity_time ) self.update_pool.spawn(self.updateRandomSite, force=True) + log.info("keepAliveThread stopped") # Periodic reloading of tracker files def reloadTrackerFilesThread(self): @@ -506,19 +520,24 @@ class FileServer(ConnectionServer): # This should probably be more sophisticated. # We should check if the files have actually changed, # and do it more often. + log.info("reloadTrackerFilesThread started") interval = 60 * 10 while self.running: self.sleep(interval) if self.stopping: break config.loadTrackersFile() + log.info("reloadTrackerFilesThread stopped") # Detects if computer back from wakeup - def wakeupWatcher(self): + def wakeupWatcherThread(self): + log.info("wakeupWatcherThread started") last_time = time.time() last_my_ips = socket.gethostbyname_ex('')[2] while self.running: self.sleep(30) + if self.stopping: + break is_time_changed = time.time() - max(self.last_request, last_time) > 60 * 3 if is_time_changed: # If taken more than 3 minute then the computer was in sleep mode @@ -543,6 +562,7 @@ class FileServer(ConnectionServer): last_time = time.time() last_my_ips = my_ips + log.info("wakeupWatcherThread stopped") # Bind and start serving sites # If passive_mode is False, FileServer starts the full-featured file serving: @@ -581,20 +601,20 @@ class FileServer(ConnectionServer): self.getSites() if not passive_mode: - self.spawn(self.updateSites) - thread_reaload_tracker_files = self.spawn(self.reloadTrackerFilesThread) + thread_keep_alive = self.spawn(self.keepAliveThread) + thread_wakeup_watcher = self.spawn(self.wakeupWatcherThread) + thread_reload_tracker_files = self.spawn(self.reloadTrackerFilesThread) thread_sites_maintenance_full = self.spawn(self.sitesMaintenanceThread, mode="full") thread_sites_maintenance_short = self.spawn(self.sitesMaintenanceThread, mode="short") - thread_keep_alive = self.spawn(self.keepAliveThread) - thread_wakeup_watcher = self.spawn(self.wakeupWatcher) + self.sleep(0.1) + self.spawn(self.updateSites) self.sleep(0.1) self.spawn(self.updateSites, check_files=True) - ConnectionServer.listen(self) - log.debug("Stopped.") + log.info("Stopped.") def stop(self): if self.running and self.portchecker.upnp_port_opened: diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index 9d93ccfd..6cd0545c 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -166,7 +166,7 @@ class UiServer: self.log.error("Web interface bind error, must be running already, exiting.... %s" % err) import main main.file_server.stop() - self.log.debug("Stopped.") + self.log.info("Stopped.") def stop(self): self.log.debug("Stopping...") From 5ec970adb89e0f0084b07763c171e9130804d40e Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 21 Oct 2021 02:24:16 +0700 Subject: [PATCH 080/114] Add option verify_files to Site.update() to allow the real content verification check, not just the simple file size-based one Add more informative updateWebsocket() notification in Site.update() and SiteStorage.verifyFiles() --- src/Site/Site.py | 19 +++++++++++++++---- src/Site/SiteStorage.py | 39 +++++++++++++++++++++++++++++++++++---- src/Ui/UiWebsocket.py | 4 ++-- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index 536f3faa..9e333751 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -689,18 +689,28 @@ class Site(object): # Update content.json from peers and download changed files # Return: None @util.Noparallel() - def update(self, announce=False, check_files=False, since=None): + def update(self, announce=False, check_files=False, verify_files=False, since=None): self.content_manager.loadContent("content.json", load_includes=False) # Reload content.json self.content_updated = None # Reset content updated time - if check_files: + if verify_files: + check_files = True + + self.updateWebsocket(updating=True) + if verify_files: + self.updateWebsocket(verifying=True) + elif check_files: + self.updateWebsocket(checking=True) + + if verify_files: + self.storage.updateBadFiles(quick_check=False) + elif check_files: self.storage.updateBadFiles(quick_check=True) # Quick check and mark bad files based on file size if not self.isServing(): + self.updateWebsocket(updated=True) return False - self.updateWebsocket(updating=True) - # Remove files that no longer in content.json self.checkBadFiles() @@ -1573,6 +1583,7 @@ class Site(object): param = None for ws in self.websockets: ws.event("siteChanged", self, param) + time.sleep(0.001) def messageWebsocket(self, message, type="info", progress=None): for ws in self.websockets: diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index c12a80b0..4e532788 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -24,6 +24,25 @@ thread_pool_fs_read = ThreadPool.ThreadPool(config.threads_fs_read, name="FS rea thread_pool_fs_write = ThreadPool.ThreadPool(config.threads_fs_write, name="FS write") thread_pool_fs_batch = ThreadPool.ThreadPool(1, name="FS batch") +class VerifyFiles_Notificator(object): + def __init__(self, site, quick_check): + self.site = site + self.quick_check = quick_check + self.scanned_files = 0 + self.websocket_update_interval = 0.25 + self.websocket_update_time = time.time() + + def inc(self): + self.scanned_files += 1 + if self.websocket_update_time + self.websocket_update_interval < time.time(): + self.send() + + def send(self): + self.websocket_update_time = time.time() + if self.quick_check: + self.site.updateWebsocket(checking=self.scanned_files) + else: + self.site.updateWebsocket(verifying=self.scanned_files) @PluginManager.acceptPlugins class SiteStorage(object): @@ -427,11 +446,14 @@ class SiteStorage(object): i = 0 self.log.debug("Verifing files...") + notificator = VerifyFiles_Notificator(self.site, quick_check) + if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first self.log.debug("VerifyFile content.json not exists") self.site.needFile("content.json", update=True) # Force update to fix corrupt file self.site.content_manager.loadContent() # Reload content.json - for content_inner_path, content in list(self.site.content_manager.contents.items()): + for content_inner_path, content in self.site.content_manager.contents.iteritems(): + notificator.inc() back["num_content"] += 1 i += 1 if i % 50 == 0: @@ -442,6 +464,7 @@ class SiteStorage(object): bad_files.append(content_inner_path) for file_relative_path in list(content.get("files", {}).keys()): + notificator.inc() back["num_file"] += 1 file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir file_inner_path = file_inner_path.strip("/") # Strip leading / @@ -452,14 +475,19 @@ class SiteStorage(object): bad_files.append(file_inner_path) continue + err = None + if quick_check: - ok = os.path.getsize(file_path) == content["files"][file_relative_path]["size"] + file_size = os.path.getsize(file_path) + expected_size = content["files"][file_relative_path]["size"] + ok = file_size == expected_size if not ok: - err = "Invalid size" + err = "Invalid size: %s - actual, %s - expected" % (file_size, expected_size) else: try: ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb")) - except Exception as err: + except Exception as err2: + err = err2 ok = False if not ok: @@ -472,6 +500,7 @@ class SiteStorage(object): optional_added = 0 optional_removed = 0 for file_relative_path in list(content.get("files_optional", {}).keys()): + notificator.inc() back["num_optional"] += 1 file_node = content["files_optional"][file_relative_path] file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir @@ -516,6 +545,8 @@ class SiteStorage(object): (content_inner_path, len(content["files"]), quick_check, optional_added, optional_removed) ) + notificator.send() + self.site.content_manager.contents.db.processDelayed() time.sleep(0.001) # Context switch to avoid gevent hangs return back diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index 88e395d6..85fa904d 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -912,9 +912,9 @@ class UiWebsocket(object): self.response(to, "ok") # Update site content.json - def actionSiteUpdate(self, to, address, check_files=False, since=None, announce=False): + def actionSiteUpdate(self, to, address, check_files=False, verify_files=False, since=None, announce=False): def updateThread(): - site.update(announce=announce, check_files=check_files, since=since) + site.update(announce=announce, check_files=check_files, verify_files=verify_files, since=since) self.response(to, "Updated") site = self.server.sites.get(address) From 5744e40505c5ab6daad597da5aadaf85dedb04c6 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 21 Oct 2021 13:19:10 +0700 Subject: [PATCH 081/114] Redesign the scheduling of site checking and verification Save `check_files_timestamp` and `verify_files_timestamp` in sites.json and run checks only when the time interval is expired. --- src/File/FileServer.py | 101 ++++++++++++++++++++++++++++---------- src/Site/Site.py | 109 +++++++++++++++++++---------------------- 2 files changed, 124 insertions(+), 86 deletions(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 73c86bd4..a0f16b97 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -303,12 +303,12 @@ class FileServer(ConnectionServer): log.debug("Checking randomly chosen site: %s", site.address_short) - self.updateSite(site, force=force) + self.updateSite(site) - def updateSite(self, site, check_files=False, force=False, dry_run=False): + def updateSite(self, site, check_files=False, verify_files=False): if not site: return False - return site.considerUpdate(check_files=check_files, force=force, dry_run=dry_run) + return site.update2(check_files=check_files, verify_files=verify_files) def invalidateUpdateTime(self, invalid_interval): for address in self.getSiteAddresses(): @@ -316,24 +316,30 @@ class FileServer(ConnectionServer): if site: site.invalidateUpdateTime(invalid_interval) - def updateSites(self, check_files=False): - self.recheckPort() + def isSiteUpdateTimeValid(self, site_address): + site = self.getSite(site_address) + if not site: + return False + return site.isUpdateTimeValid() + def updateSites(self): task_nr = self.update_sites_task_next_nr self.update_sites_task_next_nr += 1 - task_description = "updateSites [#%d, check_files=%s]" % (task_nr, check_files) + task_description = "updateSites [#%d]" % task_nr log.info("%s: started", task_description) # Don't wait port opening on first startup. Do the instant check now. if len(self.getSites()) <= 2: for address, site in list(self.getSites().items()): - self.updateSite(site, check_files=check_files) + self.updateSite(site, check_files=True) + + self.recheckPort() all_site_addresses = self.getSiteAddresses() site_addresses = [ address for address in all_site_addresses - if self.updateSite(self.getSite(address), check_files=check_files, dry_run=True) + if not self.isSiteUpdateTimeValid(address) ] log.info("%s: chosen %d sites (of %d)", task_description, len(site_addresses), len(all_site_addresses)) @@ -346,33 +352,21 @@ class FileServer(ConnectionServer): # Check sites integrity for site_address in site_addresses: - if check_files: - self.sleep(10) - else: - self.sleep(1) + site = None + self.sleep(1) + self.waitForInternetOnline() if self.stopping: break site = self.getSite(site_address) - if not self.updateSite(site, check_files=check_files, dry_run=True): + if not site or site.isUpdateTimeValid(): sites_skipped += 1 continue sites_processed += 1 - while self.running: - self.waitForInternetOnline() - if self.stopping: - break - - thread = self.update_pool.spawn(self.updateSite, site, check_files=check_files) - if check_files: - # Limit the concurency - # ZeroNet may be laggy when running from HDD. - thread.join(timeout=60) - if not self.waitForInternetOnline(): - break + thread = self.update_pool.spawn(self.updateSite, site) if self.stopping: break @@ -398,6 +392,60 @@ class FileServer(ConnectionServer): else: log.info("%s: finished in %.2fs", task_description, time.time() - start_time) + def peekSiteForVerification(self): + check_files_interval = 60 * 60 * 24 + verify_files_interval = 60 * 60 * 24 * 10 + site_addresses = self.getSiteAddresses() + random.shuffle(site_addresses) + for site_address in site_addresses: + site = self.getSite(site_address) + if not site: + continue + mode = site.isFileVerificationExpired(check_files_interval, verify_files_interval) + if mode: + return (site_address, mode) + return (None, None) + + + def sitesVerificationThread(self): + log.info("sitesVerificationThread started") + short_timeout = 10 + long_timeout = 60 + + self.sleep(long_timeout) + + while self.running: + site = None + self.sleep(short_timeout) + + if self.stopping: + break + + site_address, mode = self.peekSiteForVerification() + if not site_address: + self.sleep(long_timeout) + continue + + site = self.getSite(site_address) + if not site: + continue + + if mode == "verify": + check_files = False + verify_files = True + elif mode == "check": + check_files = True + verify_files = False + else: + continue + + log.info("running <%s> for %s" % (mode, site.address_short)) + + thread = self.update_pool.spawn(self.updateSite, site, + check_files=check_files, verify_files=verify_files) + + log.info("sitesVerificationThread stopped") + def sitesMaintenanceThread(self, mode="full"): log.info("sitesMaintenanceThread(%s) started" % mode) @@ -603,14 +651,13 @@ class FileServer(ConnectionServer): if not passive_mode: thread_keep_alive = self.spawn(self.keepAliveThread) thread_wakeup_watcher = self.spawn(self.wakeupWatcherThread) + thread_sites_verification = self.spawn(self.sitesVerificationThread) thread_reload_tracker_files = self.spawn(self.reloadTrackerFilesThread) thread_sites_maintenance_full = self.spawn(self.sitesMaintenanceThread, mode="full") thread_sites_maintenance_short = self.spawn(self.sitesMaintenanceThread, mode="short") self.sleep(0.1) self.spawn(self.updateSites) - self.sleep(0.1) - self.spawn(self.updateSites, check_files=True) ConnectionServer.listen(self) diff --git a/src/Site/Site.py b/src/Site/Site.py index 9e333751..aa2bc66b 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -183,7 +183,6 @@ class Site(object): self.worker_manager = WorkerManager(self) # Handle site download from other peers self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept) self.content_updated = None # Content.js update time - self.last_check_files_time = 0 self.last_online_update = 0 self.startup_announce_done = 0 self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout] @@ -227,6 +226,10 @@ class Site(object): settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address) if settings: self.settings = settings + if "check_files_timestamp" not in settings: + settings["check_files_timestamp"] = 0 + if "verify_files_timestamp" not in settings: + settings["verify_files_timestamp"] = 0 if "cache" not in settings: settings["cache"] = {} if "size_files_optional" not in settings: @@ -242,8 +245,17 @@ class Site(object): self.bad_files[inner_path] = min(self.bad_files[inner_path], 20) else: self.settings = { - "own": False, "serving": True, "permissions": [], "cache": {"bad_files": {}}, "size_files_optional": 0, - "added": int(time.time()), "downloaded": None, "optional_downloaded": 0, "size_optional": 0 + "check_files_timestamp": 0, + "verify_files_timestamp": 0, + "own": False, + "serving": True, + "permissions": [], + "cache": {"bad_files": {}}, + "size_files_optional": 0, + "added": int(time.time()), + "downloaded": None, + "optional_downloaded": 0, + "size_optional": 0 } # Default if config.download_optional == "auto": self.settings["autodownloadoptional"] = True @@ -281,6 +293,29 @@ class Site(object): def getSizeLimit(self): return self.settings.get("size_limit", int(config.size_limit)) + def isFileVerificationExpired(self, check_files_interval, verify_files_interval): + now = time.time() + check_files_timestamp = self.settings.get("check_files_timestamp", 0) + verify_files_timestamp = self.settings.get("verify_files_timestamp", 0) + + if check_files_interval is None: + check_files_expiration = now + 1 + else: + check_files_expiration = check_files_timestamp + check_files_interval + + if verify_files_interval is None: + verify_files_expiration = now + 1 + else: + verify_files_expiration = verify_files_timestamp + verify_files_interval + + if verify_files_expiration < now: + return "verify" + + if check_files_expiration < now: + return "check" + + return False + # Next size limit based on current size def getNextSizeLimit(self): size_limits = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000] @@ -690,9 +725,10 @@ class Site(object): # Return: None @util.Noparallel() def update(self, announce=False, check_files=False, verify_files=False, since=None): + online = self.connection_server.isInternetOnline() + self.content_manager.loadContent("content.json", load_includes=False) # Reload content.json self.content_updated = None # Reset content updated time - if verify_files: check_files = True @@ -704,8 +740,11 @@ class Site(object): if verify_files: self.storage.updateBadFiles(quick_check=False) + self.settings["check_files_timestamp"] = time.time() + self.settings["verify_files_timestamp"] = time.time() elif check_files: self.storage.updateBadFiles(quick_check=True) # Quick check and mark bad files based on file size + self.settings["check_files_timestamp"] = time.time() if not self.isServing(): self.updateWebsocket(updated=True) @@ -738,66 +777,18 @@ class Site(object): self.sendMyHashfield() self.updateHashfield() - self.refreshUpdateTime(valid=self.connection_server.isInternetOnline()) - - self.updateWebsocket(updated=True) - - @util.Noparallel(queue=True, ignore_args=True) - def _considerUpdate_realJob(self, check_files=False, force=False): - if not self._considerUpdate_check(check_files=check_files, force=force, log_reason=True): - return False - - online = self.connection_server.isInternetOnline() - - # TODO: there should be a configuration options controlling: - # * whether to check files on the program startup - # * whether to check files during the run time and how often - check_files = check_files and (self.last_check_files_time == 0) - self.last_check_files_time = time.time() - - if len(self.peers) < 50: - self.announce(mode="update") - online = online and self.connection_server.isInternetOnline() - - self.update(check_files=check_files) - online = online and self.connection_server.isInternetOnline() self.refreshUpdateTime(valid=online) - return True + self.updateWebsocket(updated=True) - def _considerUpdate_check(self, check_files=False, force=False, log_reason=False): - if not self.isServing(): - return False + # To be called from FileServer + @util.Noparallel(queue=True, ignore_args=True) + def update2(self, check_files=False, verify_files=False): + if len(self.peers) < 50: + self.announce(mode="update") - online = self.connection_server.isInternetOnline() - - run_update = False - msg = None - - if force: - run_update = True - msg = "forcing site update" - elif not online: - run_update = True - msg = "network connection seems broken, trying to update a site to check if the network is up" - elif check_files: - run_update = True - msg = "checking site's files..." - elif not self.isUpdateTimeValid(): - run_update = True - msg = "update time is not valid, updating now..." - - if run_update and log_reason: - self.log.debug(msg) - - return run_update - - def considerUpdate(self, check_files=False, force=False, dry_run=False): - run_update = self._considerUpdate_check(check_files=check_files, force=force) - if run_update and not dry_run: - run_update = self._considerUpdate_realJob(check_files=check_files, force=force) - return run_update + self.update(check_files=check_files, verify_files=verify_files) # Update site by redownload all content.json def redownloadContents(self): From b194eb0f33cd9e1f0c47a125daf6eee4f776c13c Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 21 Oct 2021 14:37:53 +0700 Subject: [PATCH 082/114] Rename param to reflect its meaning: check_site_on_reconnect -> update_site_on_reconnect --- src/Site/Site.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index aa2bc66b..d1751be3 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -1254,7 +1254,7 @@ class Site(object): return connected - def bringConnections(self, need=1, check_site_on_reconnect=False, pex=True, try_harder=False): + def bringConnections(self, need=1, update_site_on_reconnect=False, pex=True, try_harder=False): connected = len(self.getConnectedPeers()) connected_before = connected @@ -1264,16 +1264,16 @@ class Site(object): connected += self.tryConnectingToMorePeers(more=(need-connected), pex=pex, try_harder=try_harder) self.log.debug( "Connected before: %s, after: %s. Check site: %s." % - (connected_before, connected, check_site_on_reconnect) + (connected_before, connected, update_site_on_reconnect) ) - if check_site_on_reconnect and connected_before == 0 and connected > 0 and self.connection_server.has_internet: + if update_site_on_reconnect and connected_before == 0 and connected > 0 and self.connection_server.has_internet: self.greenlet_manager.spawn(self.update, check_files=False) return connected # Keep connections - def needConnections(self, num=None, check_site_on_reconnect=False, pex=True): + def needConnections(self, num=None, update_site_on_reconnect=False, pex=True): if not self.connection_server.allowsCreatingConnections(): return @@ -1284,14 +1284,14 @@ class Site(object): connected = self.bringConnections( need=need, - check_site_on_reconnect=check_site_on_reconnect, + update_site_on_reconnect=update_site_on_reconnect, pex=pex, try_harder=False) if connected < need: self.greenlet_manager.spawnLater(1.0, self.bringConnections, need=need, - check_site_on_reconnect=check_site_on_reconnect, + update_site_on_reconnect=update_site_on_reconnect, pex=pex, try_harder=True) @@ -1470,7 +1470,7 @@ class Site(object): if not startup: self.cleanupPeers() - self.needConnections(check_site_on_reconnect=True) + self.needConnections(update_site_on_reconnect=True) with gevent.Timeout(10, exception=False): self.announcer.announcePex() From cd3262a2a793046c76878e6b089d116ce6be48af Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 21 Oct 2021 14:50:54 +0700 Subject: [PATCH 083/114] code style fix in Peer.py --- src/Peer/Peer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 2e809b26..6e0d57af 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -82,7 +82,7 @@ class Peer(object): # This is to be used to prevent disconnecting from peers when doing # a periodic cleanup. def markProtected(self, interval=60*20): - self.protected = time.time() + interval + self.protected = max(self.protected, time.time() + interval) def isProtected(self): if self.protected > 0: From ddc4861223f0308917b4cd64e41480a55859f349 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 21 Oct 2021 16:18:40 +0700 Subject: [PATCH 084/114] Site.py: code cleanup --- src/Site/Site.py | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index d1751be3..ba3c9812 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -882,6 +882,28 @@ class Site(object): return False + def getPeersForForegroundPublishing(self, limit): + # Wait for some peers to appear + self.waitForPeers(limit, limit / 2, 10) # some of them... + self.waitForPeers(1, 1, 60) # or at least one... + + peers = self.getConnectedPeers() + random.shuffle(peers) + + # Prefer newer clients. + # Trying to deliver foreground updates to the latest version clients, + # expecting that they have better networking facilities. + # Note: background updates SHOULD NOT discriminate peers by their rev number, + # otherwise it can cause troubles in delivering updates to older versions. + peers = sorted(peers, key=lambda peer: peer.connection.handshake.get("rev", 0) < config.rev - 100) + + # Add more, non-connected peers if necessary + if len(peers) < limit * 2 and len(self.peers) > len(peers): + peers += self.getRecentPeers(limit * 2) + peers = set(peers) + + return peers + # Update content.json on peers @util.Noparallel() def publish(self, limit="default", inner_path="content.json", diffs={}, cb_progress=None): @@ -892,26 +914,16 @@ class Site(object): limit = 5 threads = limit - self.waitForPeers(limit, limit / 2, 10) - self.waitForPeers(1, 1, 60) - - peers = self.getConnectedPeers() - num_connected_peers = len(peers) - - random.shuffle(peers) - # Prefer newer clients - peers = sorted(peers, key=lambda peer: peer.connection.handshake.get("rev", 0) < config.rev - 100) - - # Add more, non-connected peers if necessary - if len(peers) < limit * 2 and len(self.peers) > len(peers): - peers += self.getRecentPeers(limit * 2) - peers = set(peers) + peers = self.getPeersForForegroundPublishing(limit) self.log.info("Publishing %s to %s/%s peers (connected: %s) diffs: %s (%.2fk)..." % ( - inner_path, limit, len(self.peers), num_connected_peers, list(diffs.keys()), float(len(str(diffs))) / 1024 + inner_path, + limit, len(self.peers), len(self.getConnectedPeers()), + list(diffs.keys()), float(len(str(diffs))) / 1024 )) if not peers: + self.addBackgroundPublisher(published=published, limit=limit, inner_path=inner_path, diffs=diffs) return 0 # No peers found event_done = gevent.event.AsyncResult() @@ -925,7 +937,7 @@ class Site(object): if len(published) == 0: gevent.joinall(publishers) # No successful publish, wait for all publisher - # Publish more peers in the backgroup + # Publish to more peers in the background self.log.info( "Published %s to %s peers, publishing to more peers in the background" % (inner_path, len(published)) From c36cba79806726c014eb904dbe7b1554ace6f5e7 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Thu, 21 Oct 2021 18:45:08 +0700 Subject: [PATCH 085/114] Implement new websocket command serverSetPassiveMode --- src/Connection/ConnectionServer.py | 14 ++--- src/File/FileServer.py | 97 ++++++++++++++++++++++-------- src/Ui/UiWebsocket.py | 13 ++++ 3 files changed, 90 insertions(+), 34 deletions(-) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index ad834c54..bf95a21a 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -142,28 +142,24 @@ class ConnectionServer(object): prev_sizes = {} for i in range(60): sizes = {} - total_size = 0 - for name, pool in self.managed_pools.items(): + for name, pool in list(self.managed_pools.items()): pool.join(timeout=1) size = len(pool) - sizes[name] = size - total_size += size + if size: + sizes[name] = size - if total_size == 0: + if len(sizes) == 0: break if prev_sizes != sizes: s = "" for name, size in sizes.items(): s += "%s pool: %s, " % (name, size) - s += "total: %s" % total_size - self.log.info("Waiting for tasks in managed pools to stop: %s", s) - prev_sizes = sizes - for name, pool in self.managed_pools.items(): + for name, pool in list(self.managed_pools.items()): size = len(pool) if size: self.log.info("Killing %s tasks in %s pool", size, name) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index a0f16b97..5991adf0 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -35,10 +35,15 @@ class FileServer(ConnectionServer): self.recheck_port = True + self.active_mode_thread_pool = gevent.pool.Pool(None) + self.update_pool = gevent.pool.Pool(5) self.update_start_time = 0 self.update_sites_task_next_nr = 1 + self.passive_mode = None + self.active_mode_threads = {} + self.supported_ip_types = ["ipv4"] # Outgoing ip_type support if helper.getIpType(ip) == "ipv6" or self.isIpv6Supported(): self.supported_ip_types.append("ipv6") @@ -64,7 +69,8 @@ class FileServer(ConnectionServer): ConnectionServer.__init__(self, ip, port, self.handleRequest) log.debug("Supported IP types: %s" % self.supported_ip_types) - self.managed_pools["update"] = self.pool + self.managed_pools["active_mode_thread"] = self.active_mode_thread_pool + self.managed_pools["update"] = self.update_pool if ip_type == "dual" and ip == "::": # Also bind to ipv4 addres in dual mode @@ -356,7 +362,7 @@ class FileServer(ConnectionServer): self.sleep(1) self.waitForInternetOnline() - if self.stopping: + if not self.inActiveMode(): break site = self.getSite(site_address) @@ -368,7 +374,7 @@ class FileServer(ConnectionServer): thread = self.update_pool.spawn(self.updateSite, site) - if self.stopping: + if not self.inActiveMode(): break if time.time() - progress_print_time > 60: @@ -387,7 +393,7 @@ class FileServer(ConnectionServer): time_left ) - if self.stopping: + if not self.inActiveMode(): log.info("%s: stopped", task_description) else: log.info("%s: finished in %.2fs", task_description, time.time() - start_time) @@ -414,11 +420,11 @@ class FileServer(ConnectionServer): self.sleep(long_timeout) - while self.running: + while self.inActiveMode(): site = None self.sleep(short_timeout) - if self.stopping: + if not self.inActiveMode(): break site_address, mode = self.peekSiteForVerification() @@ -457,10 +463,10 @@ class FileServer(ConnectionServer): long_timeout = min_long_timeout short_cycle_time_limit = 60 * 2 - while self.running: + while self.inActiveMode(): self.sleep(long_timeout) - if self.stopping: + if not self.inActiveMode(): break start_time = time.time() @@ -477,7 +483,7 @@ class FileServer(ConnectionServer): sites_processed = 0 for site_address in site_addresses: - if self.stopping: + if not self.inActiveMode(): break site = self.getSite(site_address) @@ -536,13 +542,13 @@ class FileServer(ConnectionServer): # performing the update for a random site. It's way better than just # silly pinging a random peer for no profit. log.info("keepAliveThread started") - while self.running: + while self.inActiveMode(): self.waitForInternetOnline() threshold = self.internet_outage_threshold / 2.0 self.sleep(threshold / 2.0) - if self.stopping: + if not self.inActiveMode(): break last_activity_time = max( @@ -570,9 +576,9 @@ class FileServer(ConnectionServer): # and do it more often. log.info("reloadTrackerFilesThread started") interval = 60 * 10 - while self.running: + while self.inActiveMode(): self.sleep(interval) - if self.stopping: + if not self.inActiveMode(): break config.loadTrackersFile() log.info("reloadTrackerFilesThread stopped") @@ -582,9 +588,9 @@ class FileServer(ConnectionServer): log.info("wakeupWatcherThread started") last_time = time.time() last_my_ips = socket.gethostbyname_ex('')[2] - while self.running: + while self.inActiveMode(): self.sleep(30) - if self.stopping: + if not self.inActiveMode(): break is_time_changed = time.time() - max(self.last_request, last_time) > 60 * 3 if is_time_changed: @@ -612,6 +618,53 @@ class FileServer(ConnectionServer): last_my_ips = my_ips log.info("wakeupWatcherThread stopped") + def killActiveModeThreads(self): + for key, thread in list(self.active_mode_threads.items()): + if thread: + if not thread.ready(): + self.log.info("killing %s" % key) + gevent.kill(thread) + del self.active_mode_threads[key] + + def setPassiveMode(self, passive_mode): + if passive_mode: + self.leaveActiveMode(); + else: + self.enterActiveMode(); + + def leaveActiveMode(self): + if self.passive_mode: + return + log.info("passive mode is ON"); + self.passive_mode = True + + def enterActiveMode(self): + if not self.passive_mode and self.passive_mode is not None: + return + log.info("passive mode is OFF"); + self.passive_mode = False + self.killActiveModeThreads() + x = self.active_mode_threads + p = self.active_mode_thread_pool + x["thread_keep_alive"] = p.spawn(self.keepAliveThread) + x["thread_wakeup_watcher"] = p.spawn(self.wakeupWatcherThread) + x["thread_sites_verification"] = p.spawn(self.sitesVerificationThread) + x["thread_reload_tracker_files"] = p.spawn(self.reloadTrackerFilesThread) + x["thread_sites_maintenance_full"] = p.spawn(self.sitesMaintenanceThread, mode="full") + x["thread_sites_maintenance_short"] = p.spawn(self.sitesMaintenanceThread, mode="short") + x["thread_initial_site_updater"] = p.spawn(self.updateSites) + + # Returns True, if an active mode thread should keep going, + # i.e active mode is enabled and the server not going to shutdown + def inActiveMode(self): + if self.passive_mode: + return False + if not self.running: + return False + if self.stopping: + return False + return True + # Bind and start serving sites # If passive_mode is False, FileServer starts the full-featured file serving: # * Checks for updates at startup. @@ -648,16 +701,7 @@ class FileServer(ConnectionServer): # Remove this line when self.sites gets completely unused self.getSites() - if not passive_mode: - thread_keep_alive = self.spawn(self.keepAliveThread) - thread_wakeup_watcher = self.spawn(self.wakeupWatcherThread) - thread_sites_verification = self.spawn(self.sitesVerificationThread) - thread_reload_tracker_files = self.spawn(self.reloadTrackerFilesThread) - thread_sites_maintenance_full = self.spawn(self.sitesMaintenanceThread, mode="full") - thread_sites_maintenance_short = self.spawn(self.sitesMaintenanceThread, mode="short") - - self.sleep(0.1) - self.spawn(self.updateSites) + self.setPassiveMode(passive_mode) ConnectionServer.listen(self) @@ -672,4 +716,7 @@ class FileServer(ConnectionServer): except Exception as err: log.info("Failed at attempt to use upnp to close port: %s" % err) + self.leaveActiveMode(); + gevent.joinall(self.active_mode_threads.values(), timeout=15) + return ConnectionServer.stop(self) diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index 85fa904d..77e5b12f 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -318,6 +318,7 @@ class UiWebsocket(object): back["updatesite"] = config.updatesite back["dist_type"] = config.dist_type back["lib_verify_best"] = CryptBitcoin.lib_verify_best + back["passive_mode"] = file_server.passive_mode return back def formatAnnouncerInfo(self, site): @@ -1164,6 +1165,18 @@ class UiWebsocket(object): file_server.portCheck() self.response(to, file_server.port_opened) + @flag.admin + @flag.no_multiuser + def actionServerSetPassiveMode(self, to, passive_mode=False): + import main + file_server = main.file_server + file_server.setPassiveMode(passive_mode) + if passive_mode: + self.cmd("notification", ["info", _["Passive mode enabled"], 5000]) + else: + self.cmd("notification", ["info", _["Passive mode disabled"], 5000]) + self.server.updateWebsocket() + @flag.admin @flag.no_multiuser def actionServerShutdown(self, to, restart=False): From 77d2d6937616a65b329bf99ed4a0d12a18ed8e44 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 22 Oct 2021 00:27:47 +0700 Subject: [PATCH 086/114] Replace `file_server.sites` with `file_server.getSites()` in ChartCollector.py --- plugins/Chart/ChartCollector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/Chart/ChartCollector.py b/plugins/Chart/ChartCollector.py index 215c603c..ceb16350 100644 --- a/plugins/Chart/ChartCollector.py +++ b/plugins/Chart/ChartCollector.py @@ -27,7 +27,7 @@ class ChartCollector(object): collectors = {} import main file_server = main.file_server - sites = file_server.sites + sites = file_server.getSites() if not sites: return collectors content_db = list(sites.values())[0].content_manager.contents.db @@ -102,7 +102,7 @@ class ChartCollector(object): def getUniquePeers(self): import main - sites = main.file_server.sites + sites = main.file_server.getSites() return set(itertools.chain.from_iterable( [site.peers.keys() for site in sites.values()] )) @@ -171,7 +171,7 @@ class ChartCollector(object): collectors = self.getCollectors() site_collectors = self.getSiteCollectors() import main - sites = main.file_server.sites + sites = main.file_server.getSites() i = 0 while 1: self.collectGlobal(collectors, self.last_values) From e3daa09316bc099e7d7db646f98c1c6a125aaf9e Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 22 Oct 2021 00:30:42 +0700 Subject: [PATCH 087/114] Improve the file server shutdown logic and display the shutdown progress bar in the UI --- plugins/AnnounceLocal/AnnounceLocalPlugin.py | 4 +- src/Connection/ConnectionServer.py | 55 +++++++--- src/Content/ContentManager.py | 2 +- src/Debug/DebugHook.py | 34 ++++-- src/File/FileServer.py | 9 +- src/Site/Site.py | 107 ++++++++++++------- src/Site/SiteStorage.py | 6 +- src/Ui/UiWebsocket.py | 2 +- src/util/GreenletManager.py | 26 ++++- 9 files changed, 170 insertions(+), 75 deletions(-) diff --git a/plugins/AnnounceLocal/AnnounceLocalPlugin.py b/plugins/AnnounceLocal/AnnounceLocalPlugin.py index b9225966..01202774 100644 --- a/plugins/AnnounceLocal/AnnounceLocalPlugin.py +++ b/plugins/AnnounceLocal/AnnounceLocalPlugin.py @@ -131,10 +131,10 @@ class FileServerPlugin(object): gevent.spawn(self.local_announcer.start) return super(FileServerPlugin, self).start(*args, **kwargs) - def stop(self): + def stop(self, ui_websocket=None): if self.local_announcer: self.local_announcer.stop() - res = super(FileServerPlugin, self).stop() + res = super(FileServerPlugin, self).stop(ui_websocket=ui_websocket) return res diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index bf95a21a..f4358965 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -131,33 +131,52 @@ class ConnectionServer(object): return False self.log.debug("Stopped.") - def stop(self): + def stop(self, ui_websocket=None): self.log.debug("Stopping %s" % self.stream_server) self.stopping = True self.running = False self.stopping_event.set() - self.onStop() + self.onStop(ui_websocket=ui_websocket) - def onStop(self): - prev_sizes = {} - for i in range(60): + def onStop(self, ui_websocket=None): + timeout = 30 + start_time = time.time() + join_quantum = 0.1 + prev_msg = None + while True: + if time.time() >= start_time + timeout: + break + + total_size = 0 sizes = {} - + timestep = 0 for name, pool in list(self.managed_pools.items()): - pool.join(timeout=1) + timestep += join_quantum + pool.join(timeout=join_quantum) size = len(pool) if size: sizes[name] = size + total_size += size if len(sizes) == 0: break - if prev_sizes != sizes: - s = "" - for name, size in sizes.items(): - s += "%s pool: %s, " % (name, size) - self.log.info("Waiting for tasks in managed pools to stop: %s", s) - prev_sizes = sizes + if timestep < 1: + time.sleep(1 - timestep) + + # format message + s = "" + for name, size in sizes.items(): + s += "%s pool: %s, " % (name, size) + msg = "Waiting for tasks in managed pools to stop: %s" % s + # Prevent flooding to log + if msg != prev_msg: + prev_msg = msg + self.log.info("%s", msg) + + percent = 100 * (time.time() - start_time) / timeout + msg = "File Server: waiting for %s tasks to stop" % total_size + self.sendShutdownProgress(ui_websocket, msg, percent) for name, pool in list(self.managed_pools.items()): size = len(pool) @@ -165,12 +184,20 @@ class ConnectionServer(object): self.log.info("Killing %s tasks in %s pool", size, name) pool.kill() + self.sendShutdownProgress(ui_websocket, "File Server stopped. Now to exit.", 100) + if self.thread_checker: gevent.kill(self.thread_checker) self.thread_checker = None if self.stream_server: self.stream_server.stop() + def sendShutdownProgress(self, ui_websocket, message, progress): + if not ui_websocket: + return + ui_websocket.cmd("progress", ["shutdown", message, progress]) + time.sleep(0.01) + # Sleeps the specified amount of time or until ConnectionServer is stopped def sleep(self, t): if t: @@ -178,7 +205,7 @@ class ConnectionServer(object): else: time.sleep(t) - # Spawns a thread that will be waited for on server being stooped (and killed after a timeout) + # Spawns a thread that will be waited for on server being stopped (and killed after a timeout) def spawn(self, *args, **kwargs): thread = self.thread_pool.spawn(*args, **kwargs) return thread diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index 7d1263ef..c6a64750 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -239,7 +239,7 @@ class ContentManager(object): if num_removed_bad_files > 0: self.site.worker_manager.removeSolvedFileTasks(mark_as_good=False) - gevent.spawn(self.site.update, since=0) + self.site.spawn(self.site.update, since=0) self.log.debug("Archived removed contents: %s, removed bad files: %s" % (num_removed_contents, num_removed_bad_files)) diff --git a/src/Debug/DebugHook.py b/src/Debug/DebugHook.py index d100a3b8..4a5bfd75 100644 --- a/src/Debug/DebugHook.py +++ b/src/Debug/DebugHook.py @@ -11,20 +11,34 @@ from . import Debug last_error = None -def shutdown(reason="Unknown"): - logging.info("Shutting down (reason: %s)..." % reason) +thread_shutdown = None + +def shutdownThread(): import main - if "file_server" in dir(main): - try: - gevent.spawn(main.file_server.stop) - if "ui_server" in dir(main): - gevent.spawn(main.ui_server.stop) - except Exception as err: - print("Proper shutdown error: %s" % err) - sys.exit(0) + try: + if "file_server" in dir(main): + thread = gevent.spawn(main.file_server.stop) + thread.join(timeout=60) + if "ui_server" in dir(main): + thread = gevent.spawn(main.ui_server.stop) + thread.join(timeout=10) + except Exception as err: + print("Error in shutdown thread: %s" % err) + sys.exit(0) else: sys.exit(0) + +def shutdown(reason="Unknown"): + global thread_shutdown + logging.info("Shutting down (reason: %s)..." % reason) + try: + if not thread_shutdown: + thread_shutdown = gevent.spawn(shutdownThread) + except Exception as err: + print("Proper shutdown error: %s" % err) + sys.exit(0) + # Store last error, ignore notify, allow manual error logging def handleError(*args, **kwargs): global last_error diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 5991adf0..6eb1ec5b 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -36,6 +36,7 @@ class FileServer(ConnectionServer): self.recheck_port = True self.active_mode_thread_pool = gevent.pool.Pool(None) + self.site_pool = gevent.pool.Pool(None) self.update_pool = gevent.pool.Pool(5) self.update_start_time = 0 @@ -71,6 +72,7 @@ class FileServer(ConnectionServer): self.managed_pools["active_mode_thread"] = self.active_mode_thread_pool self.managed_pools["update"] = self.update_pool + self.managed_pools["site"] = self.site_pool if ip_type == "dual" and ip == "::": # Also bind to ipv4 addres in dual mode @@ -707,7 +709,7 @@ class FileServer(ConnectionServer): log.info("Stopped.") - def stop(self): + def stop(self, ui_websocket=None): if self.running and self.portchecker.upnp_port_opened: log.debug('Closing port %d' % self.port) try: @@ -716,7 +718,4 @@ class FileServer(ConnectionServer): except Exception as err: log.info("Failed at attempt to use upnp to close port: %s" % err) - self.leaveActiveMode(); - gevent.joinall(self.active_mode_threads.values(), timeout=15) - - return ConnectionServer.stop(self) + return ConnectionServer.stop(self, ui_websocket=ui_websocket) diff --git a/src/Site/Site.py b/src/Site/Site.py index ba3c9812..5b228ff9 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -175,25 +175,8 @@ class Site(object): self.fzs_count = random.randint(0, self.fzs_range / 4) self.fzs_timestamp = 0 - self.content = None # Load content.json - self.peers = {} # Key: ip:port, Value: Peer.Peer - self.peers_recent = collections.deque(maxlen=150) - self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself) - self.greenlet_manager = GreenletManager.GreenletManager() # Running greenlets - self.worker_manager = WorkerManager(self) # Handle site download from other peers - self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept) - self.content_updated = None # Content.js update time - self.last_online_update = 0 - self.startup_announce_done = 0 - self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout] - self.page_requested = False # Page viewed in browser - self.websockets = [] # Active site websocket connections - + ############################################## self.connection_server = None - self.loadSettings(settings) # Load settings from sites.json - self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files - self.content_manager = ContentManager(self) - self.content_manager.loadContents() # Load content.json files if "main" in sys.modules: # import main has side-effects, breaks tests import main if "file_server" in dir(main): # Use global file server by default if possible @@ -203,6 +186,26 @@ class Site(object): self.connection_server = main.file_server else: self.connection_server = FileServer() + ############################################## + + self.content = None # Load content.json + self.peers = {} # Key: ip:port, Value: Peer.Peer + self.peers_recent = collections.deque(maxlen=150) + self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself) + self.greenlet_manager = GreenletManager.GreenletManager(self.connection_server.site_pool) # Running greenlets + self.worker_manager = WorkerManager(self) # Handle site download from other peers + self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept) + self.content_updated = None # Content.js update time + self.last_online_update = 0 + self.startup_announce_done = 0 + self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout] + self.page_requested = False # Page viewed in browser + self.websockets = [] # Active site websocket connections + + self.loadSettings(settings) # Load settings from sites.json + self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files + self.content_manager = ContentManager(self) + self.content_manager.loadContents() # Load content.json files self.announcer = SiteAnnouncer(self) # Announce and get peer list from other nodes @@ -275,14 +278,32 @@ class Site(object): SiteManager.site_manager.load(False) SiteManager.site_manager.saveDelayed() + # Returns True if any site-related activity should be interrupted + # due to connection server being stooped or site being deleted + def isStopping(self): + return self.connection_server.stopping or self.settings.get("deleting", False) + + # Returns False if any network activity for the site should not happen def isServing(self): if config.offline: return False - elif self.connection_server.stopping: + elif self.isStopping(): return False else: return self.settings["serving"] + # Spawns a thread that will be waited for on server being stopped (and killed after a timeout). + # Short cut to self.greenlet_manager.spawn() + def spawn(self, *args, **kwargs): + thread = self.greenlet_manager.spawn(*args, **kwargs) + return thread + + # Spawns a thread that will be waited for on server being stopped (and killed after a timeout). + # Short cut to self.greenlet_manager.spawnLater() + def spawnLater(self, *args, **kwargs): + thread = self.greenlet_manager.spawnLater(*args, **kwargs) + return thread + def getSettingsCache(self): back = {} back["bad_files"] = self.bad_files @@ -418,7 +439,7 @@ class Site(object): # Optionals files if inner_path == "content.json": - gevent.spawn(self.updateHashfield) + self.spawn(self.updateHashfield) for file_relative_path in list(self.content_manager.contents[inner_path].get("files_optional", {}).keys()): file_inner_path = content_inner_dir + file_relative_path @@ -437,7 +458,7 @@ class Site(object): include_threads = [] for file_relative_path in list(self.content_manager.contents[inner_path].get("includes", {}).keys()): file_inner_path = content_inner_dir + file_relative_path - include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer) + include_thread = self.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer) include_threads.append(include_thread) if config.verbose: @@ -517,9 +538,9 @@ class Site(object): ) if self.isAddedRecently(): - gevent.spawn(self.announce, mode="start", force=True) + self.spawn(self.announce, mode="start", force=True) else: - gevent.spawn(self.announce, mode="update") + self.spawn(self.announce, mode="update") if check_size: # Check the size first valid = self.downloadContent("content.json", download_files=False) # Just download content.json files @@ -615,7 +636,7 @@ class Site(object): self.log.info("CheckModifications: %s: %s > %s" % ( inner_path, res["modified_files"][inner_path], my_modified.get(inner_path, 0) )) - t = gevent.spawn(self.pooledDownloadContent, modified_contents, only_if_bad=True) + t = self.spawn(self.pooledDownloadContent, modified_contents, only_if_bad=True) threads.append(t) if send_back: @@ -628,7 +649,7 @@ class Site(object): self.log.info("CheckModifications: %s: %s < %s" % ( inner_path, res["modified_files"][inner_path], my_modified.get(inner_path, 0) )) - gevent.spawn(self.publisher, inner_path, [peer], [], 1) + self.spawn(self.publisher, inner_path, [peer], [], 1) self.log.debug("CheckModifications: Waiting for %s pooledDownloadContent" % len(threads)) gevent.joinall(threads) @@ -685,7 +706,7 @@ class Site(object): updaters = [] for i in range(updater_limit): - updaters.append(gevent.spawn(self.updater, peers_try, queried, need_queries, since)) + updaters.append(self.spawn(self.updater, peers_try, queried, need_queries, since)) for r in range(10): gevent.joinall(updaters, timeout=5+r) @@ -738,13 +759,17 @@ class Site(object): elif check_files: self.updateWebsocket(checking=True) - if verify_files: - self.storage.updateBadFiles(quick_check=False) - self.settings["check_files_timestamp"] = time.time() - self.settings["verify_files_timestamp"] = time.time() - elif check_files: - self.storage.updateBadFiles(quick_check=True) # Quick check and mark bad files based on file size - self.settings["check_files_timestamp"] = time.time() + if check_files: + if verify_files: + self.storage.updateBadFiles(quick_check=False) # Full-featured checksum verification + else: + self.storage.updateBadFiles(quick_check=True) # Quick check and mark bad files based on file size + # Don't update the timestamps in case of the application being shut down, + # so we can make another try next time. + if not self.isStopping(): + self.settings["check_files_timestamp"] = time.time() + if verify_files: + self.settings["verify_files_timestamp"] = time.time() if not self.isServing(): self.updateWebsocket(updated=True) @@ -766,7 +791,7 @@ class Site(object): if self.bad_files: self.log.debug("Bad files: %s" % self.bad_files) - gevent.spawn(self.retryBadFiles, force=True) + self.spawn(self.retryBadFiles, force=True) if len(queried) == 0: # Failed to query modifications @@ -856,7 +881,7 @@ class Site(object): background_publisher = BackgroundPublisher(self, published=published, limit=limit, inner_path=inner_path, diffs=diffs) self.background_publishers[inner_path] = background_publisher - gevent.spawn(background_publisher.process) + self.spawn(background_publisher.process) def processBackgroundPublishers(self): with self.background_publishers_lock: @@ -928,7 +953,7 @@ class Site(object): event_done = gevent.event.AsyncResult() for i in range(min(len(peers), limit, threads)): - publisher = gevent.spawn(self.publisher, inner_path, peers, published, limit, diffs, event_done, cb_progress) + publisher = self.spawn(self.publisher, inner_path, peers, published, limit, diffs, event_done, cb_progress) publishers.append(publisher) event_done.get() # Wait for done @@ -946,7 +971,7 @@ class Site(object): self.addBackgroundPublisher(published=published, limit=limit, inner_path=inner_path, diffs=diffs) # Send my hashfield to every connected peer if changed - gevent.spawn(self.sendMyHashfield, 100) + self.spawn(self.sendMyHashfield, 100) return len(published) @@ -1109,7 +1134,7 @@ class Site(object): if not self.content_manager.contents.get("content.json"): # No content.json, download it first! self.log.debug("Need content.json first (inner_path: %s, priority: %s)" % (inner_path, priority)) if priority > 0: - gevent.spawn(self.announce) + self.spawn(self.announce) if inner_path != "content.json": # Prevent double download task = self.worker_manager.addTask("content.json", peer) task["evt"].get() @@ -1508,6 +1533,9 @@ class Site(object): # Send hashfield to peers def sendMyHashfield(self, limit=5): + if not self.isServing(): + return False + if not self.content_manager.hashfield: # No optional files return False @@ -1525,6 +1553,9 @@ class Site(object): # Update hashfield def updateHashfield(self, limit=5): + if not self.isServing(): + return False + # Return if no optional files if not self.content_manager.hashfield and not self.content_manager.has_optional_files: return False diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index 4e532788..b89aedbf 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -375,7 +375,7 @@ class SiteStorage(object): # Reopen DB to check changes if self.has_db: self.closeDb("New dbschema") - gevent.spawn(self.getDb) + self.site.spawn(self.getDb) elif not config.disable_db and should_load_to_db and self.has_db: # Load json file to db if config.verbose: self.log.debug("Loading json file to db: %s (file: %s)" % (inner_path, file)) @@ -458,6 +458,10 @@ class SiteStorage(object): i += 1 if i % 50 == 0: time.sleep(0.001) # Context switch to avoid gevent hangs + + if self.site.isStopping(): + break + if not os.path.isfile(self.getPath(content_inner_path)): # Missing content.json file back["num_content_missing"] += 1 self.log.debug("[MISSING] %s" % content_inner_path) diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index 77e5b12f..7e68eb34 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -1187,7 +1187,7 @@ class UiWebsocket(object): return False if restart: main.restart_after_shutdown = True - main.file_server.stop() + main.file_server.stop(ui_websocket=self) main.ui_server.stop() if restart: diff --git a/src/util/GreenletManager.py b/src/util/GreenletManager.py index e024233d..d711d09a 100644 --- a/src/util/GreenletManager.py +++ b/src/util/GreenletManager.py @@ -3,17 +3,37 @@ from Debug import Debug class GreenletManager: - def __init__(self): + # pool is either gevent.pool.Pool or GreenletManager. + # if pool is None, new gevent.pool.Pool() is created. + def __init__(self, pool=None): self.greenlets = set() + if not pool: + pool = gevent.pool.Pool(None) + self.pool = pool + + def _spawn_later(self, seconds, *args, **kwargs): + # If pool is another GreenletManager, delegate to it. + if hasattr(self.pool, 'spawnLater'): + return self.pool.spawnLater(seconds, *args, **kwargs) + + # There's gevent.spawn_later(), but there isn't gevent.pool.Pool.spawn_later(). + # Doing manually. + greenlet = self.pool.greenlet_class(*args, **kwargs) + self.pool.add(greenlet) + greenlet.start_later(seconds) + return greenlet + + def _spawn(self, *args, **kwargs): + return self.pool.spawn(*args, **kwargs) def spawnLater(self, *args, **kwargs): - greenlet = gevent.spawn_later(*args, **kwargs) + greenlet = self._spawn_later(*args, **kwargs) greenlet.link(lambda greenlet: self.greenlets.remove(greenlet)) self.greenlets.add(greenlet) return greenlet def spawn(self, *args, **kwargs): - greenlet = gevent.spawn(*args, **kwargs) + greenlet = self._spawn(*args, **kwargs) greenlet.link(lambda greenlet: self.greenlets.remove(greenlet)) self.greenlets.add(greenlet) return greenlet From 19b840defde0585a50cbfb08895d06f7c660825c Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 22 Oct 2021 02:59:28 +0700 Subject: [PATCH 088/114] Implement new websocket command serverSetOfflineMode --- src/Connection/ConnectionServer.py | 16 +++++- src/File/FileServer.py | 87 ++++++++++++++++++------------ src/Site/Site.py | 3 +- src/Ui/UiWebsocket.py | 26 ++++++--- 4 files changed, 89 insertions(+), 43 deletions(-) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index f4358965..e7951321 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -479,15 +479,27 @@ class ConnectionServer(object): self.had_external_incoming = False self.log.info("Internet offline") + def setOfflineMode(self, offline_mode): + if config.offline == offline_mode: + return + config.offline = offline_mode # Yep, awkward + if offline_mode: + self.log.info("offline mode is ON") + else: + self.log.info("offline mode is OFF") + + def isOfflineMode(self): + return config.offline + def allowsCreatingConnections(self): - if config.offline: + if self.isOfflineMode(): return False if self.stopping: return False return True def allowsAcceptingConnections(self): - if config.offline: + if self.isOfflineMode(): return False if self.stopping: return False diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 6eb1ec5b..e432e4a7 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -43,8 +43,10 @@ class FileServer(ConnectionServer): self.update_sites_task_next_nr = 1 self.passive_mode = None + self.active_mode = None self.active_mode_threads = {} + self.supported_ip_types = ["ipv4"] # Outgoing ip_type support if helper.getIpType(ip) == "ipv6" or self.isIpv6Supported(): self.supported_ip_types.append("ipv6") @@ -196,7 +198,7 @@ class FileServer(ConnectionServer): FileRequest = imp.load_source("FileRequest", "src/File/FileRequest.py").FileRequest def portCheck(self): - if config.offline: + if self.isOfflineMode(): log.info("Offline mode: port check disabled") res = {"ipv4": None, "ipv6": None} self.port_opened = res @@ -276,7 +278,7 @@ class FileServer(ConnectionServer): # Returns True if we've spent some time waiting for Internet # Returns None if FileServer is stopping or the Offline mode is enabled def waitForInternetOnline(self): - if config.offline or self.stopping: + if self.isOfflineMode() or self.stopping: return None if self.isInternetOnline(): @@ -284,7 +286,7 @@ class FileServer(ConnectionServer): while not self.isInternetOnline(): self.sleep(30) - if config.offline or self.stopping: + if self.isOfflineMode() or self.stopping: return None if self.isInternetOnline(): break @@ -364,7 +366,7 @@ class FileServer(ConnectionServer): self.sleep(1) self.waitForInternetOnline() - if not self.inActiveMode(): + if not self.isActiveMode(): break site = self.getSite(site_address) @@ -376,7 +378,7 @@ class FileServer(ConnectionServer): thread = self.update_pool.spawn(self.updateSite, site) - if not self.inActiveMode(): + if not self.isActiveMode(): break if time.time() - progress_print_time > 60: @@ -395,7 +397,7 @@ class FileServer(ConnectionServer): time_left ) - if not self.inActiveMode(): + if not self.isActiveMode(): log.info("%s: stopped", task_description) else: log.info("%s: finished in %.2fs", task_description, time.time() - start_time) @@ -422,11 +424,11 @@ class FileServer(ConnectionServer): self.sleep(long_timeout) - while self.inActiveMode(): + while self.isActiveMode(): site = None self.sleep(short_timeout) - if not self.inActiveMode(): + if not self.isActiveMode(): break site_address, mode = self.peekSiteForVerification() @@ -465,10 +467,10 @@ class FileServer(ConnectionServer): long_timeout = min_long_timeout short_cycle_time_limit = 60 * 2 - while self.inActiveMode(): + while self.isActiveMode(): self.sleep(long_timeout) - if not self.inActiveMode(): + if not self.isActiveMode(): break start_time = time.time() @@ -485,7 +487,7 @@ class FileServer(ConnectionServer): sites_processed = 0 for site_address in site_addresses: - if not self.inActiveMode(): + if not self.isActiveMode(): break site = self.getSite(site_address) @@ -544,13 +546,13 @@ class FileServer(ConnectionServer): # performing the update for a random site. It's way better than just # silly pinging a random peer for no profit. log.info("keepAliveThread started") - while self.inActiveMode(): + while self.isActiveMode(): self.waitForInternetOnline() threshold = self.internet_outage_threshold / 2.0 self.sleep(threshold / 2.0) - if not self.inActiveMode(): + if not self.isActiveMode(): break last_activity_time = max( @@ -578,9 +580,9 @@ class FileServer(ConnectionServer): # and do it more often. log.info("reloadTrackerFilesThread started") interval = 60 * 10 - while self.inActiveMode(): + while self.isActiveMode(): self.sleep(interval) - if not self.inActiveMode(): + if not self.isActiveMode(): break config.loadTrackersFile() log.info("reloadTrackerFilesThread stopped") @@ -590,9 +592,9 @@ class FileServer(ConnectionServer): log.info("wakeupWatcherThread started") last_time = time.time() last_my_ips = socket.gethostbyname_ex('')[2] - while self.inActiveMode(): + while self.isActiveMode(): self.sleep(30) - if not self.inActiveMode(): + if not self.isActiveMode(): break is_time_changed = time.time() - max(self.last_request, last_time) > 60 * 3 if is_time_changed: @@ -620,31 +622,47 @@ class FileServer(ConnectionServer): last_my_ips = my_ips log.info("wakeupWatcherThread stopped") + def setOfflineMode(self, offline_mode): + ConnectionServer.setOfflineMode(self, offline_mode) + self.setupActiveMode() + + def setPassiveMode(self, passive_mode): + if self.passive_mode == passive_mode: + return + self.passive_mode = passive_mode + if self.passive_mode: + log.info("passive mode is ON"); + else: + log.info("passive mode is OFF"); + self.setupActiveMode() + + def isPassiveMode(self): + return self.passive_mode + + def setupActiveMode(self): + active_mode = (not self.passive_mode) and (not self.isOfflineMode()) + if self.active_mode == active_mode: + return + self.active_mode = active_mode + if self.active_mode: + log.info("active mode is ON"); + self.enterActiveMode(); + else: + log.info("active mode is OFF"); + self.leaveActiveMode(); + def killActiveModeThreads(self): for key, thread in list(self.active_mode_threads.items()): if thread: if not thread.ready(): - self.log.info("killing %s" % key) + log.info("killing %s" % key) gevent.kill(thread) del self.active_mode_threads[key] - def setPassiveMode(self, passive_mode): - if passive_mode: - self.leaveActiveMode(); - else: - self.enterActiveMode(); - def leaveActiveMode(self): - if self.passive_mode: - return - log.info("passive mode is ON"); - self.passive_mode = True + pass def enterActiveMode(self): - if not self.passive_mode and self.passive_mode is not None: - return - log.info("passive mode is OFF"); - self.passive_mode = False self.killActiveModeThreads() x = self.active_mode_threads p = self.active_mode_thread_pool @@ -658,8 +676,9 @@ class FileServer(ConnectionServer): # Returns True, if an active mode thread should keep going, # i.e active mode is enabled and the server not going to shutdown - def inActiveMode(self): - if self.passive_mode: + def isActiveMode(self): + self.setupActiveMode() + if not self.active_mode: return False if not self.running: return False diff --git a/src/Site/Site.py b/src/Site/Site.py index 5b228ff9..31acea03 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -285,7 +285,7 @@ class Site(object): # Returns False if any network activity for the site should not happen def isServing(self): - if config.offline: + if self.connection_server.isOfflineMode(): return False elif self.isStopping(): return False @@ -1375,6 +1375,7 @@ class Site(object): # Return: Recently found peers def getRecentPeers(self, need_num): + need_num = int(need_num) found = list(set(self.peers_recent)) self.log.debug( "Recent peers %s of %s (need: %s)" % diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index 7e68eb34..80d53b45 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -1170,12 +1170,26 @@ class UiWebsocket(object): def actionServerSetPassiveMode(self, to, passive_mode=False): import main file_server = main.file_server - file_server.setPassiveMode(passive_mode) - if passive_mode: - self.cmd("notification", ["info", _["Passive mode enabled"], 5000]) - else: - self.cmd("notification", ["info", _["Passive mode disabled"], 5000]) - self.server.updateWebsocket() + if file_server.isPassiveMode() != passive_mode: + file_server.setPassiveMode(passive_mode) + if file_server.isPassiveMode(): + self.cmd("notification", ["info", _["Passive mode enabled"], 5000]) + else: + self.cmd("notification", ["info", _["Passive mode disabled"], 5000]) + self.server.updateWebsocket() + + @flag.admin + @flag.no_multiuser + def actionServerSetOfflineMode(self, to, offline_mode=False): + import main + file_server = main.file_server + if file_server.isOfflineMode() != offline_mode: + file_server.setOfflineMode(offline_mode) + if file_server.isOfflineMode(): + self.cmd("notification", ["info", _["Offline mode enabled"], 5000]) + else: + self.cmd("notification", ["info", _["Offline mode disabled"], 5000]) + self.server.updateWebsocket() @flag.admin @flag.no_multiuser From 1ef129bdf9193cbbc9aa1d0f595345e6453eff52 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 22 Oct 2021 14:38:05 +0700 Subject: [PATCH 089/114] Fix "changed size during iteration" in verifyFiles() --- src/Site/SiteStorage.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index b89aedbf..97f720dc 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -439,6 +439,8 @@ class SiteStorage(object): return inner_path # Verify all files sha512sum using content.json + # The result may not be accurate if self.site.isStopping(). + # verifyFiles() return immediately in that case. def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True): bad_files = [] back = defaultdict(int) @@ -446,14 +448,44 @@ class SiteStorage(object): i = 0 self.log.debug("Verifing files...") - notificator = VerifyFiles_Notificator(self.site, quick_check) - if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first self.log.debug("VerifyFile content.json not exists") self.site.needFile("content.json", update=True) # Force update to fix corrupt file self.site.content_manager.loadContent() # Reload content.json - for content_inner_path, content in self.site.content_manager.contents.iteritems(): - notificator.inc() + + # Trying to read self.site.content_manager.contents without being stuck + # on reading the long file list and also without getting + # "RuntimeError: dictionary changed size during iteration" + # We can't use just list(iteritems()) since it loads all the contents files + # at once and gets unresponsive. + contents = {} + notificator = None + tries = 0 + max_tries = 40 + stop = False + while not stop: + try: + contents = {} + notificator = VerifyFiles_Notificator(self.site, quick_check) + for content_inner_path, content in self.site.content_manager.contents.iteritems(): + notificator.inc() + contents[content_inner_path] = content + if self.site.isStopping(): + stop = True + break + stop = True + except RuntimeError as err: + if "changed size during iteration" in str(err): + tries += 1 + if tries >= max_tries: + self.log.info("contents.json file list changed during iteration. %s tries done. Giving up.", tries) + stop = True + self.log.info("contents.json file list changed during iteration. Trying again... (%s)", tries) + time.sleep(2 * tries) + else: + stop = True + + for content_inner_path, content in contents.items(): back["num_content"] += 1 i += 1 if i % 50 == 0: From 1b68182a76e7ab9fbbb8ed82a266d9b55dfab391 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 22 Oct 2021 17:18:24 +0700 Subject: [PATCH 090/114] FileServer: don't schedule multiple updates for the same site in parallel --- src/File/FileServer.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index e432e4a7..ab669090 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -313,13 +313,34 @@ class FileServer(ConnectionServer): log.debug("Checking randomly chosen site: %s", site.address_short) - self.updateSite(site) + self.spawnUpdateSite(site).join() def updateSite(self, site, check_files=False, verify_files=False): if not site: return False return site.update2(check_files=check_files, verify_files=verify_files) + def spawnUpdateSite(self, site, check_files=False, verify_files=False): + thread = self.update_pool.spawn(self.updateSite, site, + check_files=check_files, verify_files=verify_files) + thread.site_address = site.address + return thread + + def siteIsInUpdatePool(self, site_address): + while True: + restart = False + for thread in list(iter(self.update_pool)): + if not thread.site_address: + # Possible race condition in assigning thread.site_address in spawnUpdateSite() + # Trying again. + self.sleep(0.1) + restart = True + break + if thread.site_address == site_address: + return True + if not restart: + return False + def invalidateUpdateTime(self, invalid_interval): for address in self.getSiteAddresses(): site = self.getSite(address) @@ -370,13 +391,13 @@ class FileServer(ConnectionServer): break site = self.getSite(site_address) - if not site or site.isUpdateTimeValid(): + if not site or site.isUpdateTimeValid() or self.siteIsInUpdatePool(site_address): sites_skipped += 1 continue sites_processed += 1 - thread = self.update_pool.spawn(self.updateSite, site) + thread = self.spawnUpdateSite(site) if not self.isActiveMode(): break @@ -436,6 +457,12 @@ class FileServer(ConnectionServer): self.sleep(long_timeout) continue + while self.siteIsInUpdatePool(site_address) and self.isActiveMode(): + self.sleep(1) + + if not self.isActiveMode(): + break + site = self.getSite(site_address) if not site: continue @@ -451,7 +478,7 @@ class FileServer(ConnectionServer): log.info("running <%s> for %s" % (mode, site.address_short)) - thread = self.update_pool.spawn(self.updateSite, site, + thread = self.spawnUpdateSite(site, check_files=check_files, verify_files=verify_files) log.info("sitesVerificationThread stopped") From fe24e17baa24dd364943702e8b4f701b4407f39c Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 23 Oct 2021 21:40:41 +0700 Subject: [PATCH 091/114] Fix the prev commit --- src/File/FileServer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index ab669090..16afab24 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -291,7 +291,7 @@ class FileServer(ConnectionServer): if self.isInternetOnline(): break if len(self.update_pool) == 0: - thread = self.update_pool.spawn(self.updateRandomSite) + thread = self.thread_pool.spawn(self.updateRandomSite) thread.join() self.recheckPort() @@ -330,13 +330,14 @@ class FileServer(ConnectionServer): while True: restart = False for thread in list(iter(self.update_pool)): - if not thread.site_address: + thread_site_address = getattr(thread, 'site_address', None) + if not thread_site_address: # Possible race condition in assigning thread.site_address in spawnUpdateSite() # Trying again. self.sleep(0.1) restart = True break - if thread.site_address == site_address: + if thread_site_address == site_address: return True if not restart: return False @@ -397,6 +398,7 @@ class FileServer(ConnectionServer): sites_processed += 1 + log.info("running for %s" % site.address_short) thread = self.spawnUpdateSite(site) if not self.isActiveMode(): From e612f9363118d5e53e8597a369195f100a49b9b9 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 23 Oct 2021 21:41:14 +0700 Subject: [PATCH 092/114] Spawn message loops for outgoing connections in a sepatare pool managed by ConnectionServer --- src/Connection/Connection.py | 2 +- src/Connection/ConnectionServer.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 4a7c6ab9..398a4cc3 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -167,7 +167,7 @@ class Connection(object): # Detect protocol event_connected = self.event_connected self.send({"cmd": "handshake", "req_id": 0, "params": self.getHandshakeInfo()}) - gevent.spawn(self.messageLoop) + self.server.outgoing_pool.spawn(self.messageLoop) connect_res = event_connected.get() # Wait for handshake if self.sock: self.sock.settimeout(timeout_before) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index e7951321..03a93daa 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -79,6 +79,9 @@ class ConnectionServer(object): self.pool = Pool(500) # do not accept more than 500 connections self.managed_pools["incoming"] = self.pool + self.outgoing_pool = Pool(None) + self.managed_pools["outgoing"] = self.outgoing_pool + # Bittorrent style peerid self.peer_id = "-UT3530-%s" % CryptHash.random(12, "base64") From b4f94e50221a5cb0612d1a63fb3c576284f5637b Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 23 Oct 2021 21:44:41 +0700 Subject: [PATCH 093/114] Make use of waitForPeers() when running FileServer-driven update() --- src/Site/Site.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index 31acea03..881f7d7c 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -812,6 +812,9 @@ class Site(object): def update2(self, check_files=False, verify_files=False): if len(self.peers) < 50: self.announce(mode="update") + self.waitForPeers(5, 5, 30); + self.waitForPeers(2, 2, 30); + self.waitForPeers(1, 1, 60); self.update(check_files=check_files, verify_files=verify_files) @@ -891,21 +894,36 @@ class Site(object): background_publisher.finalize() del self.background_publishers[inner_path] - def waitForPeers(self, num_peers, num_connected_peers, time_limit): + def waitForPeers_realJob(self, need_nr_peers, need_nr_connected_peers, time_limit): start_time = time.time() for _ in range(time_limit): - if len(self.peers) >= num_peers and len(self.getConnectedPeers()) >= num_connected_peers: - return True + nr_connected_peers = len(self.getConnectedPeers()) + nr_peers = len(self.peers) + if nr_peers >= need_nr_peers and nr_connected_peers >= need_nr_connected_peers: + return nr_connected_peers + self.updateWebsocket(connecting_to_peers=nr_connected_peers) self.announce(mode="more", force=True) + if not self.isServing(): + return nr_connected_peers for wait in range(10): - self.needConnections(num=num_connected_peers) + self.needConnections(num=need_nr_connected_peers) time.sleep(2) - if len(self.peers) >= num_peers and len(self.getConnectedPeers()) >= num_connected_peers: - return True + nr_connected_peers = len(self.getConnectedPeers()) + nr_peers = len(self.peers) + self.updateWebsocket(connecting_to_peers=nr_connected_peers) + if not self.isServing(): + return nr_connected_peers + if nr_peers >= need_nr_peers and nr_connected_peers >= need_nr_connected_peers: + return nr_connected_peers if time.time() - start_time > time_limit: - return True + return nr_connected_peers - return False + return nr_connected_peers + + def waitForPeers(self, need_nr_peers, need_nr_connected_peers, time_limit): + nr_connected_peers = self.waitForPeers_realJob(need_nr_peers, need_nr_connected_peers, time_limit) + self.updateWebsocket(connected_to_peers=nr_connected_peers) + return nr_connected_peers def getPeersForForegroundPublishing(self, limit): # Wait for some peers to appear From b512c54f75b4eb9eef9479a582e637be54bdd898 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sun, 24 Oct 2021 23:53:08 +0700 Subject: [PATCH 094/114] Implement new logic for waiting for connected peers when updating or publishing a site --- plugins/TrackerShare/TrackerSharePlugin.py | 2 +- src/File/FileServer.py | 30 ++- src/Peer/Peer.py | 10 +- src/Site/Site.py | 204 +++++++++---------- src/Site/SiteHelpers.py | 220 +++++++++++++++++++++ 5 files changed, 331 insertions(+), 135 deletions(-) create mode 100644 src/Site/SiteHelpers.py diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 99ebf692..4fe888fd 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -462,7 +462,7 @@ if "tracker_storage" not in locals(): class SiteAnnouncerPlugin(object): def getTrackers(self): tracker_storage.setSiteAnnouncer(self) - tracker_storage.checkDiscoveringTrackers(self.site.getConnectedPeers()) + tracker_storage.checkDiscoveringTrackers(self.site.getConnectedPeers(onlyFullyConnected=True)) trackers = super(SiteAnnouncerPlugin, self).getTrackers() shared_trackers = list(tracker_storage.getTrackers().keys()) if shared_trackers: diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 16afab24..5487d0e9 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -3,6 +3,7 @@ import time import random import socket import sys +import weakref import gevent import gevent.pool @@ -42,6 +43,8 @@ class FileServer(ConnectionServer): self.update_start_time = 0 self.update_sites_task_next_nr = 1 + self.update_threads = weakref.WeakValueDictionary() + self.passive_mode = None self.active_mode = None self.active_mode_threads = {} @@ -317,30 +320,23 @@ class FileServer(ConnectionServer): def updateSite(self, site, check_files=False, verify_files=False): if not site: - return False - return site.update2(check_files=check_files, verify_files=verify_files) + return + site.update2(check_files=check_files, verify_files=verify_files) def spawnUpdateSite(self, site, check_files=False, verify_files=False): thread = self.update_pool.spawn(self.updateSite, site, check_files=check_files, verify_files=verify_files) - thread.site_address = site.address + self.update_threads[site.address] = thread return thread + def lookupInUpdatePool(self, site_address): + thread = self.update_threads.get(site_address, None) + if not thread or thread.ready(): + return None + return thread + def siteIsInUpdatePool(self, site_address): - while True: - restart = False - for thread in list(iter(self.update_pool)): - thread_site_address = getattr(thread, 'site_address', None) - if not thread_site_address: - # Possible race condition in assigning thread.site_address in spawnUpdateSite() - # Trying again. - self.sleep(0.1) - restart = True - break - if thread_site_address == site_address: - return True - if not restart: - return False + return self.lookupInUpdatePool(site_address) is not None def invalidateUpdateTime(self, invalid_interval): for address in self.getSiteAddresses(): diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 6e0d57af..3637c822 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -54,6 +54,8 @@ class Peer(object): self.download_bytes = 0 # Bytes downloaded self.download_time = 0 # Time spent to download + self.protectedRequests = ["getFile", "streamFile", "update", "listModified"] + def __getattr__(self, key): if key == "hashfield": self.has_hashfield = True @@ -78,10 +80,8 @@ class Peer(object): logger.log(log_level, "%s:%s %s" % (self.ip, self.port, text)) - # Site marks its Peers protected, if it has not enough peers connected. - # This is to be used to prevent disconnecting from peers when doing - # a periodic cleanup. - def markProtected(self, interval=60*20): + # Protect connection from being closed by site.cleanupPeers() + def markProtected(self, interval=60*2): self.protected = max(self.protected, time.time() + interval) def isProtected(self): @@ -195,6 +195,8 @@ class Peer(object): for retry in range(1, 4): # Retry 3 times try: + if cmd in self.protectedRequests: + self.markProtected() if not self.connection: raise Exception("No connection found") res = self.connection.request(cmd, params, stream_to) diff --git a/src/Site/Site.py b/src/Site/Site.py index 881f7d7c..5929617c 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -28,6 +28,7 @@ from Plugin import PluginManager from File import FileServer from .SiteAnnouncer import SiteAnnouncer from . import SiteManager +from . import SiteHelpers class ScaledTimeoutHandler: def __init__(self, val_min, val_max, handler=None, scaler=None): @@ -145,7 +146,6 @@ class BackgroundPublisher: self.site.log.info("Background publisher: Published %s to %s peers", self.inner_path, len(self.published)) - @PluginManager.acceptPlugins class Site(object): @@ -209,6 +209,10 @@ class Site(object): self.announcer = SiteAnnouncer(self) # Announce and get peer list from other nodes + self.peer_connector = SiteHelpers.PeerConnector(self) # Connect more peers in background by request + self.persistent_peer_req = None # The persistent peer requirement, managed by maintenance handler + + if not self.settings.get("wrapper_key"): # To auth websocket permissions self.settings["wrapper_key"] = CryptHash.random() self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"]) @@ -753,7 +757,6 @@ class Site(object): if verify_files: check_files = True - self.updateWebsocket(updating=True) if verify_files: self.updateWebsocket(verifying=True) elif check_files: @@ -771,16 +774,32 @@ class Site(object): if verify_files: self.settings["verify_files_timestamp"] = time.time() + if verify_files: + self.updateWebsocket(verified=True) + elif check_files: + self.updateWebsocket(checked=True) + if not self.isServing(): - self.updateWebsocket(updated=True) return False + if announce: + self.updateWebsocket(updating=True) + self.announce(mode="update", force=True) + + reqs = [ + self.peer_connector.newReq(4, 4, 30), + self.peer_connector.newReq(2, 2, 60), + self.peer_connector.newReq(1, 1, 120) + ] + nr_connected_peers = self.waitForPeers(reqs); + if nr_connected_peers < 1: + return + + self.updateWebsocket(updating=True) + # Remove files that no longer in content.json self.checkBadFiles() - if announce: - self.announce(mode="update", force=True) - # Full update, we can reset bad files if check_files and since == 0: self.bad_files = {} @@ -810,12 +829,6 @@ class Site(object): # To be called from FileServer @util.Noparallel(queue=True, ignore_args=True) def update2(self, check_files=False, verify_files=False): - if len(self.peers) < 50: - self.announce(mode="update") - self.waitForPeers(5, 5, 30); - self.waitForPeers(2, 2, 30); - self.waitForPeers(1, 1, 60); - self.update(check_files=check_files, verify_files=verify_files) # Update site by redownload all content.json @@ -894,41 +907,13 @@ class Site(object): background_publisher.finalize() del self.background_publishers[inner_path] - def waitForPeers_realJob(self, need_nr_peers, need_nr_connected_peers, time_limit): - start_time = time.time() - for _ in range(time_limit): - nr_connected_peers = len(self.getConnectedPeers()) - nr_peers = len(self.peers) - if nr_peers >= need_nr_peers and nr_connected_peers >= need_nr_connected_peers: - return nr_connected_peers - self.updateWebsocket(connecting_to_peers=nr_connected_peers) - self.announce(mode="more", force=True) - if not self.isServing(): - return nr_connected_peers - for wait in range(10): - self.needConnections(num=need_nr_connected_peers) - time.sleep(2) - nr_connected_peers = len(self.getConnectedPeers()) - nr_peers = len(self.peers) - self.updateWebsocket(connecting_to_peers=nr_connected_peers) - if not self.isServing(): - return nr_connected_peers - if nr_peers >= need_nr_peers and nr_connected_peers >= need_nr_connected_peers: - return nr_connected_peers - if time.time() - start_time > time_limit: - return nr_connected_peers - - return nr_connected_peers - - def waitForPeers(self, need_nr_peers, need_nr_connected_peers, time_limit): - nr_connected_peers = self.waitForPeers_realJob(need_nr_peers, need_nr_connected_peers, time_limit) - self.updateWebsocket(connected_to_peers=nr_connected_peers) - return nr_connected_peers - def getPeersForForegroundPublishing(self, limit): # Wait for some peers to appear - self.waitForPeers(limit, limit / 2, 10) # some of them... - self.waitForPeers(1, 1, 60) # or at least one... + reqs = [ + self.peer_connector.newReq(limit, limit / 2, 10), # some of them... + self.peer_connector.newReq(1, 1, 60) # or at least one... + ] + self.waitForPeers(reqs) peers = self.getConnectedPeers() random.shuffle(peers) @@ -1206,6 +1191,10 @@ class Site(object): peer = Peer(ip, port, self) self.peers[key] = peer peer.found(source) + + self.peer_connector.processReqs() + self.peer_connector.addPeer(peer) + return peer def announce(self, *args, **kwargs): @@ -1288,76 +1277,54 @@ class Site(object): limit = min(limit, config.connected_limit) return limit - def tryConnectingToMorePeers(self, more=1, pex=True, try_harder=False): - max_peers = more * 2 + 10 - if try_harder: - max_peers += 10000 + ############################################################################ - connected = 0 - for peer in self.getRecentPeers(max_peers): - if not peer.isConnected(): - if pex: - peer.pex() - else: - peer.ping(timeout=2.0, tryes=1) + # Returns the maximum value of current reqs for connections + def waitingForConnections(self): + self.peer_connector.processReqs() + return self.peer_connector.need_nr_connected_peers - if peer.isConnected(): - connected += 1 - - if connected >= more: - break - - return connected - - def bringConnections(self, need=1, update_site_on_reconnect=False, pex=True, try_harder=False): - connected = len(self.getConnectedPeers()) - connected_before = connected - - self.log.debug("Need connections: %s, Current: %s, Total: %s" % (need, connected, len(self.peers))) - - if connected < need: - connected += self.tryConnectingToMorePeers(more=(need-connected), pex=pex, try_harder=try_harder) - self.log.debug( - "Connected before: %s, after: %s. Check site: %s." % - (connected_before, connected, update_site_on_reconnect) - ) - - if update_site_on_reconnect and connected_before == 0 and connected > 0 and self.connection_server.has_internet: - self.greenlet_manager.spawn(self.update, check_files=False) - - return connected - - # Keep connections - def needConnections(self, num=None, update_site_on_reconnect=False, pex=True): + def needConnections(self, num=None, update_site_on_reconnect=False): if not self.connection_server.allowsCreatingConnections(): return if num is None: num = self.getPreferableActiveConnectionCount() + num = min(len(self.peers), num) - need = min(len(self.peers), num) + req = self.peer_connector.newReq(0, num) + return req - connected = self.bringConnections( - need=need, - update_site_on_reconnect=update_site_on_reconnect, - pex=pex, - try_harder=False) + # Wait for peers to ne known and/or connected and send updates to the UI + def waitForPeers(self, reqs): + if not reqs: + return 0 + i = 0 + nr_connected_peers = -1 + while self.isServing(): + ready_reqs = list(filter(lambda req: req.ready(), reqs)) + if len(ready_reqs) == len(reqs): + if nr_connected_peers < 0: + nr_connected_peers = ready_reqs[0].nr_connected_peers + break + waiting_reqs = list(filter(lambda req: not req.ready(), reqs)) + if not waiting_reqs: + break + waiting_req = waiting_reqs[0] + #self.log.debug("waiting_req: %s %s %s", waiting_req.need_nr_connected_peers, waiting_req.nr_connected_peers, waiting_req.expiration_interval) + waiting_req.waitHeartbeat(timeout=1.0) + if i > 0 and nr_connected_peers != waiting_req.nr_connected_peers: + nr_connected_peers = waiting_req.nr_connected_peers + self.updateWebsocket(connecting_to_peers=nr_connected_peers) + i += 1 + self.updateWebsocket(connected_to_peers=max(nr_connected_peers, 0)) + if i > 1: + # If we waited some time, pause now for displaying connected_to_peers message in the UI. + # This sleep is solely needed for site status updates on ZeroHello to be more cool-looking. + gevent.sleep(1) + return nr_connected_peers - if connected < need: - self.greenlet_manager.spawnLater(1.0, self.bringConnections, - need=need, - update_site_on_reconnect=update_site_on_reconnect, - pex=pex, - try_harder=True) - - if connected < num: - self.markConnectedPeersProtected() - - return connected - - def markConnectedPeersProtected(self): - for peer in self.getConnectedPeers(): - peer.markProtected() + ############################################################################ # Return: Probably peers verified to be connectable recently def getConnectablePeers(self, need_num=5, ignore=[], allow_private=True): @@ -1429,15 +1396,26 @@ class Site(object): return found[0:need_num] - def getConnectedPeers(self): + # Returns the list of connected peers + # By default the result may contain peers chosen optimistically: + # If the connection is being established and 20 seconds have not yet passed + # since the connection start time, those peers are included in the result. + # Set onlyFullyConnected=True for restricting only by fully connected peers. + def getConnectedPeers(self, onlyFullyConnected=False): back = [] if not self.connection_server: return [] tor_manager = self.connection_server.tor_manager for connection in self.connection_server.connections: + if len(back) >= len(self.peers): # short cut for breaking early; no peers to check left + break + if not connection.connected and time.time() - connection.start_time > 20: # Still not connected after 20s continue + if not connection.connected and onlyFullyConnected: # Only fully connected peers + continue + peer = self.peers.get("%s:%s" % (connection.ip, connection.port)) if peer: if connection.ip.endswith(".onion") and connection.target_onion and tor_manager.start_onions: @@ -1479,8 +1457,8 @@ class Site(object): def cleanupPeers(self): self.removeDeadPeers() - limit = self.getActiveConnectionCountLimit() - connected_peers = [peer for peer in self.getConnectedPeers() if peer.isConnected()] # Only fully connected peers + limit = max(self.getActiveConnectionCountLimit(), self.waitingForConnections()) + connected_peers = self.getConnectedPeers(onlyFullyConnected=True) need_to_close = len(connected_peers) - limit if need_to_close > 0: @@ -1526,10 +1504,10 @@ class Site(object): if not startup: self.cleanupPeers() - self.needConnections(update_site_on_reconnect=True) + self.persistent_peer_req = self.needConnections(update_site_on_reconnect=True) + self.persistent_peer_req.result_connected.wait(timeout=2.0) - with gevent.Timeout(10, exception=False): - self.announcer.announcePex() + #self.announcer.announcePex() self.processBackgroundPublishers() @@ -1559,7 +1537,7 @@ class Site(object): return False sent = 0 - connected_peers = self.getConnectedPeers() + connected_peers = self.getConnectedPeers(onlyFullyConnected=True) for peer in connected_peers: if peer.sendMyHashfield(): sent += 1 @@ -1581,7 +1559,7 @@ class Site(object): s = time.time() queried = 0 - connected_peers = self.getConnectedPeers() + connected_peers = self.getConnectedPeers(onlyFullyConnected=True) for peer in connected_peers: if peer.time_hashfield: continue diff --git a/src/Site/SiteHelpers.py b/src/Site/SiteHelpers.py new file mode 100644 index 00000000..65f0530b --- /dev/null +++ b/src/Site/SiteHelpers.py @@ -0,0 +1,220 @@ +import time +import weakref +import gevent + +class ConnectRequirement(object): + next_id = 1 + def __init__(self, need_nr_peers, need_nr_connected_peers, expiration_interval=None): + self.need_nr_peers = need_nr_peers # how many total peers we need + self.need_nr_connected_peers = need_nr_connected_peers # how many connected peers we need + self.result = gevent.event.AsyncResult() # resolves on need_nr_peers condition + self.result_connected = gevent.event.AsyncResult() # resolves on need_nr_connected_peers condition + + self.expiration_interval = expiration_interval + self.expired = False + if expiration_interval: + self.expire_at = time.time() + expiration_interval + else: + self.expire_at = None + + self.nr_peers = -1 # updated PeerConnector() + self.nr_connected_peers = -1 # updated PeerConnector() + + self.heartbeat = gevent.event.AsyncResult() + + self.id = type(self).next_id + type(self).next_id += 1 + + def fulfilled(self): + return self.result.ready() and self.result_connected.ready() + + def ready(self): + return self.expired or self.fulfilled() + + # Heartbeat send when any of the following happens: + # * self.result is set + # * self.result_connected is set + # * self.nr_peers changed + # * self.nr_peers_connected changed + # * self.expired is set + def waitHeartbeat(self, timeout=None): + if self.heartbeat.ready(): + self.heartbeat = gevent.event.AsyncResult() + return self.heartbeat.wait(timeout=timeout) + + def sendHeartbeat(self): + self.heartbeat.set_result() + if self.heartbeat.ready(): + self.heartbeat = gevent.event.AsyncResult() + +class PeerConnector(object): + + def __init__(self, site): + self.site = site + + self.peer_reqs = weakref.WeakValueDictionary() # How many connected peers we need. + # Separate entry for each requirement. + # Objects of type ConnectRequirement. + self.peer_connector_controller = None # Thread doing the orchestration in background. + self.peer_connector_workers = dict() # Threads trying to connect to individual peers. + self.peer_connector_worker_limit = 5 # Max nr of workers. + self.peer_connector_announcer = None # Thread doing announces in background. + + # Max effective values. Set by processReqs(). + self.need_nr_peers = 0 + self.need_nr_connected_peers = 0 + self.nr_peers = 0 # set by processReqs() + self.nr_connected_peers = 0 # set by processReqs2() + + self.peers = list() + + def addReq(self, req): + self.peer_reqs[req.id] = req + self.processReqs() + + def newReq(self, need_nr_peers, need_nr_connected_peers, expiration_interval=None): + req = ConnectRequirement(need_nr_peers, need_nr_connected_peers, expiration_interval=expiration_interval) + self.addReq(req) + return req + + def processReqs(self, nr_connected_peers=None): + nr_peers = len(self.site.peers) + self.nr_peers = nr_peers + + need_nr_peers = 0 + need_nr_connected_peers = 0 + + items = list(self.peer_reqs.items()) + for key, req in items: + send_heartbeat = False + + if req.expire_at and req.expire_at < time.time(): + req.expired = True + self.peer_reqs.pop(key, None) + send_heartbeat = True + elif req.result.ready() and req.result_connected.ready(): + pass + else: + if nr_connected_peers is not None: + if req.need_nr_peers <= nr_peers and req.need_nr_connected_peers <= nr_connected_peers: + req.result.set_result(nr_peers) + req.result_connected.set_result(nr_connected_peers) + send_heartbeat = True + if req.nr_peers != nr_peers or req.nr_connected_peers != nr_connected_peers: + req.nr_peers = nr_peers + req.nr_connected_peers = nr_connected_peers + send_heartbeat = True + + if not (req.result.ready() and req.result_connected.ready()): + need_nr_peers = max(need_nr_peers, req.need_nr_peers) + need_nr_connected_peers = max(need_nr_connected_peers, req.need_nr_connected_peers) + + if send_heartbeat: + req.sendHeartbeat() + + self.need_nr_peers = need_nr_peers + self.need_nr_connected_peers = need_nr_connected_peers + + if nr_connected_peers is None: + nr_connected_peers = 0 + if need_nr_peers > nr_peers: + self.spawnPeerConnectorAnnouncer(); + if need_nr_connected_peers > nr_connected_peers: + self.spawnPeerConnectorController(); + + def processReqs2(self): + self.nr_connected_peers = len(self.site.getConnectedPeers(onlyFullyConnected=True)) + self.processReqs(nr_connected_peers=self.nr_connected_peers) + + # For adding new peers when ConnectorController is working. + # While it is iterating over a cached list of peers, there can be a significant lag + # for a newly discovered peer to get in sight of the controller. + # Suppose most previously known peers are dead and we've just get a few + # new peers from a tracker. + # So we mix the new peer to the cached list. + # When ConnectorController is stopped (self.peers is empty), we just do nothing here. + def addPeer(self, peer): + if not self.peers: + return + if peer not in self.peers: + self.peers.append(peer) + + def keepGoing(self): + return self.site.isServing() and self.site.connection_server.allowsCreatingConnections() + + def peerConnectorWorker(self, peer): + if not peer.isConnected(): + peer.connect() + if peer.isConnected(): + self.processReqs2() + + def peerConnectorController(self): + self.peers = list() + addendum = 20 + while self.keepGoing(): + + if len(self.site.peers) < 1: + # No peers and no way to manage this from this method. + # Just give up. + break + + self.processReqs2() + + if self.need_nr_connected_peers <= self.nr_connected_peers: + # Ok, nobody waits for connected peers. + # Done. + break + + if len(self.peers) < 1: + # refill the peer list + self.peers = self.site.getRecentPeers(self.need_nr_connected_peers * 2 + addendum) + addendum = addendum * 2 + 50 + if len(self.peers) <= self.nr_connected_peers: + # looks like all known peers are connected + # start announcePex() in background and give up + self.site.announcer.announcePex() + break + + # try connecting to peers + while self.keepGoing() and len(self.peer_connector_workers) < self.peer_connector_worker_limit: + if len(self.peers) < 1: + break + + peer = self.peers.pop(0) + + if peer.isConnected(): + continue + + thread = self.peer_connector_workers.get(peer, None) + if thread: + continue + + thread = self.site.spawn(self.peerConnectorWorker, peer) + self.peer_connector_workers[peer] = thread + thread.link(lambda thread, peer=peer: self.peer_connector_workers.pop(peer, None)) + + # wait for more room in self.peer_connector_workers + while self.keepGoing() and len(self.peer_connector_workers) >= self.peer_connector_worker_limit: + gevent.sleep(2) + + self.peers = list() + self.peer_connector_controller = None + + def peerConnectorAnnouncer(self): + while self.keepGoing(): + if self.need_nr_peers <= self.nr_peers: + break + self.site.announce(mode="more") + self.processReqs2() + if self.need_nr_peers <= self.nr_peers: + break + gevent.sleep(10) + self.peer_connector_announcer = None + + def spawnPeerConnectorController(self): + if self.peer_connector_controller is None or self.peer_connector_controller.ready(): + self.peer_connector_controller = self.site.spawn(self.peerConnectorController) + + def spawnPeerConnectorAnnouncer(self): + if self.peer_connector_announcer is None or self.peer_connector_announcer.ready(): + self.peer_connector_announcer = self.site.spawn(self.peerConnectorAnnouncer) From 1fd1f47a9410331ea80aed37cde9d3f4743a8e7c Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 25 Oct 2021 12:53:37 +0700 Subject: [PATCH 095/114] Fix detection of the broken Internet connection on the app start up --- src/Connection/Connection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 398a4cc3..504daf58 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -118,6 +118,8 @@ class Connection(object): # Open connection to peer and wait for handshake def connect(self): + self.updateOnlineStatus(outgoing_activity=True) + if not self.event_connected or self.event_connected.ready(): self.event_connected = gevent.event.AsyncResult() From ce971ab738e8ee938aeab9b0e927872b41a8df3d Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 25 Oct 2021 13:18:43 +0700 Subject: [PATCH 096/114] Don't increment `bad_file` failed tries counter on rediscovering the same file on update() Do increment it only on actual fileFailed() event. --- src/Site/Site.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index 5929617c..cb6630a5 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -629,7 +629,7 @@ class Site(object): if has_newer: # We dont have this file or we have older modified_contents.append(inner_path) - self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 + self.bad_files[inner_path] = self.bad_files.get(inner_path, 1) if has_older: send_back.append(inner_path) @@ -1163,7 +1163,7 @@ class Site(object): self.log.debug("%s: Download not allowed" % inner_path) return False - self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file + self.bad_files[inner_path] = self.bad_files.get(inner_path, 1) # Mark as bad file task = self.worker_manager.addTask(inner_path, peer, priority=priority, file_info=file_info) if blocking: From 8f908c961db56a463708903d86c6c5afb620c82f Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 25 Oct 2021 16:12:00 +0700 Subject: [PATCH 097/114] Fine-tuning PeerConnector --- src/Site/SiteHelpers.py | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/Site/SiteHelpers.py b/src/Site/SiteHelpers.py index 65f0530b..c3be446c 100644 --- a/src/Site/SiteHelpers.py +++ b/src/Site/SiteHelpers.py @@ -66,6 +66,7 @@ class PeerConnector(object): self.nr_peers = 0 # set by processReqs() self.nr_connected_peers = 0 # set by processReqs2() + # Connector Controller state self.peers = list() def addReq(self, req): @@ -139,6 +140,9 @@ class PeerConnector(object): if peer not in self.peers: self.peers.append(peer) + def sleep(self, t): + self.site.connection_server.sleep(t) + def keepGoing(self): return self.site.isServing() and self.site.connection_server.allowsCreatingConnections() @@ -153,10 +157,14 @@ class PeerConnector(object): addendum = 20 while self.keepGoing(): - if len(self.site.peers) < 1: - # No peers and no way to manage this from this method. - # Just give up. - break + no_peers_loop = 0 + while len(self.site.peers) < 1: + # No peers at all. + # Waiting for the announcer to discover some peers. + self.sleep(10 + no_peers_loop) + no_peers_loop += 1 + if not self.keepGoing() or no_peers_loop > 60: + break self.processReqs2() @@ -165,15 +173,19 @@ class PeerConnector(object): # Done. break + if len(self.site.peers) < 1: + break + if len(self.peers) < 1: # refill the peer list - self.peers = self.site.getRecentPeers(self.need_nr_connected_peers * 2 + addendum) - addendum = addendum * 2 + 50 + self.peers = self.site.getRecentPeers(self.need_nr_connected_peers * 2 + self.nr_connected_peers + addendum) + addendum = min(addendum * 2 + 50, 10000) if len(self.peers) <= self.nr_connected_peers: - # looks like all known peers are connected - # start announcePex() in background and give up + # Looks like all known peers are connected. + # Waiting for the announcer to discover some peers. self.site.announcer.announcePex() - break + self.sleep(10) + continue # try connecting to peers while self.keepGoing() and len(self.peer_connector_workers) < self.peer_connector_worker_limit: @@ -195,7 +207,10 @@ class PeerConnector(object): # wait for more room in self.peer_connector_workers while self.keepGoing() and len(self.peer_connector_workers) >= self.peer_connector_worker_limit: - gevent.sleep(2) + self.sleep(2) + + if not self.site.connection_server.isInternetOnline(): + self.sleep(20) self.peers = list() self.peer_connector_controller = None @@ -208,7 +223,9 @@ class PeerConnector(object): self.processReqs2() if self.need_nr_peers <= self.nr_peers: break - gevent.sleep(10) + self.sleep(10) + if not self.site.connection_server.isInternetOnline(): + self.sleep(20) self.peer_connector_announcer = None def spawnPeerConnectorController(self): From 1a8d30146ecabae3e0e23174a9918498ffd4c19d Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 25 Oct 2021 16:18:35 +0700 Subject: [PATCH 098/114] Fix typos in comments --- src/Site/Site.py | 10 +++++----- src/Site/SiteHelpers.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index cb6630a5..456d02f1 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -283,7 +283,7 @@ class Site(object): SiteManager.site_manager.saveDelayed() # Returns True if any site-related activity should be interrupted - # due to connection server being stooped or site being deleted + # due to connection server being stopped or site being deleted def isStopping(self): return self.connection_server.stopping or self.settings.get("deleting", False) @@ -353,7 +353,7 @@ class Site(object): def isAddedRecently(self): return time.time() - self.settings.get("added", 0) < 60 * 60 * 24 - # Download all file from content.json + # Download all files from content.json def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}): s = time.time() if config.verbose: @@ -1597,10 +1597,10 @@ class Site(object): # Add event listeners def addEventListeners(self): - self.onFileStart = util.Event() # If WorkerManager added new task + self.onFileStart = util.Event() # If WorkerManager added new task self.onFileDone = util.Event() # If WorkerManager successfully downloaded a file self.onFileFail = util.Event() # If WorkerManager failed to download a file - self.onComplete = util.Event() # All file finished + self.onComplete = util.Event() # All files finished self.onFileStart.append(lambda inner_path: self.fileStarted()) # No parameters to make Noparallel batching working self.onFileDone.append(lambda inner_path: self.fileDone(inner_path)) @@ -1629,7 +1629,7 @@ class Site(object): time.sleep(0.001) # Wait for other files adds self.updateWebsocket(file_started=True) - # File downloaded successful + # File downloaded successfully def fileDone(self, inner_path): # File downloaded, remove it from bad files if inner_path in self.bad_files: diff --git a/src/Site/SiteHelpers.py b/src/Site/SiteHelpers.py index c3be446c..5abcd46e 100644 --- a/src/Site/SiteHelpers.py +++ b/src/Site/SiteHelpers.py @@ -31,7 +31,7 @@ class ConnectRequirement(object): def ready(self): return self.expired or self.fulfilled() - # Heartbeat send when any of the following happens: + # Heartbeat sent when any of the following happens: # * self.result is set # * self.result_connected is set # * self.nr_peers changed From dff52d691a51b2c87d71464829a5dc3658b7c267 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 25 Oct 2021 17:09:16 +0700 Subject: [PATCH 099/114] Small improvements in FileServer --- src/File/FileServer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 5487d0e9..9ef3ae9c 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -280,6 +280,7 @@ class FileServer(ConnectionServer): # Returns False if Internet is immediately available # Returns True if we've spent some time waiting for Internet # Returns None if FileServer is stopping or the Offline mode is enabled + @util.Noparallel() def waitForInternetOnline(self): if self.isOfflineMode() or self.stopping: return None @@ -294,6 +295,7 @@ class FileServer(ConnectionServer): if self.isInternetOnline(): break if len(self.update_pool) == 0: + log.info("Internet connection seems to be broken. Running an update for a random site to check if we are able to connect to any peer.") thread = self.thread_pool.spawn(self.updateRandomSite) thread.join() @@ -314,7 +316,7 @@ class FileServer(ConnectionServer): if not site: return - log.debug("Checking randomly chosen site: %s", site.address_short) + log.info("Randomly chosen site: %s", site.address_short) self.spawnUpdateSite(site).join() @@ -446,6 +448,7 @@ class FileServer(ConnectionServer): while self.isActiveMode(): site = None self.sleep(short_timeout) + self.waitForInternetOnline() if not self.isActiveMode(): break From 93a95f511a21f06aa80bb05e3a45011bb758e283 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Mon, 25 Oct 2021 17:59:35 +0700 Subject: [PATCH 100/114] Limit the pex request frequency, interval is 120 secs for each peer --- src/Peer/Peer.py | 9 +++++- src/Site/SiteAnnouncer.py | 64 ++++++++++++++++++++++----------------- src/Site/SiteHelpers.py | 2 +- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 3637c822..72e9c47f 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -44,6 +44,7 @@ class Peer(object): self.time_response = None # Time of last successful response from peer self.time_added = time.time() self.last_ping = None # Last response time for ping + self.last_pex = 0 # Last query/response time for pex self.is_tracker_connection = False # Tracker connection instead of normal peer self.reputation = 0 # More likely to connect if larger self.last_content_json_update = 0.0 # Modify date of last received content.json @@ -305,10 +306,15 @@ class Peer(object): return response_time # Request peer exchange from peer - def pex(self, site=None, need_num=5): + def pex(self, site=None, need_num=5, request_interval=60*2): if not site: site = self.site # If no site defined request peers for this site + if self.last_pex + request_interval >= time.time(): + return False + + self.last_pex = time.time() + # give back 5 connectible peers packed_peers = helper.packPeers(self.site.getConnectablePeers(5, allow_private=False)) request = {"site": site.address, "peers": packed_peers["ipv4"], "need": need_num} @@ -317,6 +323,7 @@ class Peer(object): if packed_peers["ipv6"]: request["peers_ipv6"] = packed_peers["ipv6"] res = self.request("pex", request) + self.last_pex = time.time() if not res or "error" in res: return False added = 0 diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index 6a510583..31004d02 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -302,39 +302,47 @@ class SiteAnnouncer(object): self.updateWebsocket(trackers="announced") @util.Noparallel(blocking=False) - def announcePex(self, query_num=2, need_num=10): - if not self.site.isServing(): - return + def announcePex(self, query_num=2, need_num=10, establish_connections=True): + peers = [] + try: + peer_count = 20 + query_num * 2 - self.updateWebsocket(pex="announcing") + # Wait for some peers to connect + for _ in range(5): + if not self.site.isServing(): + return + peers = self.site.getConnectedPeers(onlyFullyConnected=True) + if len(peers) > 0: + break + time.sleep(2) - peers = self.site.getConnectedPeers() - if len(peers) == 0: # Wait 3s for connections - time.sleep(3) - peers = self.site.getConnectedPeers() + if len(peers) < peer_count and establish_connections: + # Small number of connected peers for this site, connect to any + peers = list(self.site.getRecentPeers(peer_count)) - if len(peers) == 0: # Small number of connected peers for this site, connect to any - peers = list(self.site.getRecentPeers(20)) - need_num = 10 + if len(peers) > 0: + self.updateWebsocket(pex="announcing") - random.shuffle(peers) - done = 0 - total_added = 0 - for peer in peers: - num_added = peer.pex(need_num=need_num) - if num_added is not False: - done += 1 - total_added += num_added - if num_added: - self.site.worker_manager.onPeers() - self.site.updateWebsocket(peers_added=num_added) - else: + random.shuffle(peers) + done = 0 + total_added = 0 + for peer in peers: + if not establish_connections and not peer.isConnected(): + continue + num_added = peer.pex(need_num=need_num) + if num_added is not False: + done += 1 + total_added += num_added + if num_added: + self.site.worker_manager.onPeers() + self.site.updateWebsocket(peers_added=num_added) + if done == query_num: + break time.sleep(0.1) - if done == query_num: - break - self.log.debug("Pex result: from %s peers got %s new peers." % (done, total_added)) - - self.updateWebsocket(pex="announced") + self.log.debug("Pex result: from %s peers got %s new peers." % (done, total_added)) + finally: + if len(peers) > 0: + self.updateWebsocket(pex="announced") def updateWebsocket(self, **kwargs): if kwargs: diff --git a/src/Site/SiteHelpers.py b/src/Site/SiteHelpers.py index 5abcd46e..c095ad66 100644 --- a/src/Site/SiteHelpers.py +++ b/src/Site/SiteHelpers.py @@ -183,7 +183,7 @@ class PeerConnector(object): if len(self.peers) <= self.nr_connected_peers: # Looks like all known peers are connected. # Waiting for the announcer to discover some peers. - self.site.announcer.announcePex() + self.site.announcer.announcePex(establish_connections=False) self.sleep(10) continue From 645f3ba34ab37b508d4da7d0d222d963d7908a3c Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 26 Oct 2021 17:38:40 +0700 Subject: [PATCH 101/114] Reorganization of Peer class and peer-related Site's methods --- plugins/TrackerShare/TrackerSharePlugin.py | 2 +- src/Connection/Connection.py | 9 +- src/Connection/ConnectionServer.py | 14 +++ src/Peer/Peer.py | 66 +++++++++--- src/Site/Site.py | 118 ++++++++++----------- src/Site/SiteAnnouncer.py | 2 +- src/Site/SiteHelpers.py | 22 +++- 7 files changed, 150 insertions(+), 83 deletions(-) diff --git a/plugins/TrackerShare/TrackerSharePlugin.py b/plugins/TrackerShare/TrackerSharePlugin.py index 4fe888fd..42c79422 100644 --- a/plugins/TrackerShare/TrackerSharePlugin.py +++ b/plugins/TrackerShare/TrackerSharePlugin.py @@ -462,7 +462,7 @@ if "tracker_storage" not in locals(): class SiteAnnouncerPlugin(object): def getTrackers(self): tracker_storage.setSiteAnnouncer(self) - tracker_storage.checkDiscoveringTrackers(self.site.getConnectedPeers(onlyFullyConnected=True)) + tracker_storage.checkDiscoveringTrackers(self.site.getConnectedPeers(only_fully_connected=True)) trackers = super(SiteAnnouncerPlugin, self).getTrackers() shared_trackers = list(tracker_storage.getTrackers().keys()) if shared_trackers: diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 504daf58..40519b7f 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -124,12 +124,13 @@ class Connection(object): self.event_connected = gevent.event.AsyncResult() self.type = "out" + + unreachability = self.server.getIpUnreachability(self.ip) + if unreachability: + raise Exception(unreachability) + if self.ip_type == "onion": - if not self.server.tor_manager or not self.server.tor_manager.enabled: - raise Exception("Can't connect to onion addresses, no Tor controller present") self.sock = self.server.tor_manager.createSocket(self.ip, self.port) - elif config.tor == "always" and helper.isPrivateIp(self.ip) and self.ip not in config.ip_local: - raise Exception("Can't connect to local IPs in Tor: always mode") elif config.trackers_proxy != "disable" and config.tor != "always" and self.is_tracker_connection: if config.trackers_proxy == "tor": self.sock = self.server.tor_manager.createSocket(self.ip, self.port) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 03a93daa..16834ff5 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -519,3 +519,17 @@ class ConnectionServer(object): mid = int(len(corrections) / 2 - 1) median = (corrections[mid - 1] + corrections[mid] + corrections[mid + 1]) / 3 return median + + # Checks if a network address can be reachable in the current configuration + # and returs a string describing why it cannot. + # If the network address can be reachable, returns False. + def getIpUnreachability(self, ip): + ip_type = helper.getIpType(ip) + if ip_type == 'onion' and not self.tor_manager.enabled: + return "Can't connect to onion addresses, no Tor controller present" + if config.tor == "always" and helper.isPrivateIp(ip) and ip not in config.ip_local: + return "Can't connect to local IPs in Tor: always mode" + return False + + def isIpReachable(self, ip): + return self.getIpUnreachability(ip) == False diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 72e9c47f..28a7220b 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -32,6 +32,10 @@ class Peer(object): self.site = site self.key = "%s:%s" % (ip, port) + self.ip_type = helper.getIpType(ip) + + self.removed = False + self.log_level = logging.DEBUG self.connection_error_log_level = logging.DEBUG @@ -41,7 +45,7 @@ class Peer(object): self.time_hashfield = None # Last time peer's hashfiled downloaded self.time_my_hashfield_sent = None # Last time my hashfield sent to peer self.time_found = time.time() # Time of last found in the torrent tracker - self.time_response = None # Time of last successful response from peer + self.time_response = 0 # Time of last successful response from peer self.time_added = time.time() self.last_ping = None # Last response time for ping self.last_pex = 0 # Last query/response time for pex @@ -49,6 +53,7 @@ class Peer(object): self.reputation = 0 # More likely to connect if larger self.last_content_json_update = 0.0 # Modify date of last received content.json self.protected = 0 + self.reachable = False self.connection_error = 0 # Series of connection error self.hash_failed = 0 # Number of bad files from peer @@ -57,6 +62,8 @@ class Peer(object): self.protectedRequests = ["getFile", "streamFile", "update", "listModified"] + self.updateReachable() + def __getattr__(self, key): if key == "hashfield": self.has_hashfield = True @@ -97,7 +104,36 @@ class Peer(object): return self.connection and self.connection.connected def isTtlExpired(self, ttl): - return (time.time() - self.time_found) > ttl + last_activity = max(self.time_found, self.time_response) + return (time.time() - last_activity) > ttl + + def isReachable(self): + return self.reachable + + def updateReachable(self): + connection_server = self.getConnectionServer() + if not self.port: + self.reachable = False + else: + self.reachable = connection_server.isIpReachable(self.ip) + + # Peer proved to to be connectable recently + def isConnectable(self): + if self.connection_error >= 1: # The last connection attempt failed + return False + if time.time() - self.time_response > 60 * 60 * 2: # Last successful response more than 2 hours ago + return False + return self.isReachable() + + def getConnectionServer(self): + if self.connection_server: + connection_server = self.connection_server + elif self.site: + connection_server = self.site.connection_server + else: + import main + connection_server = main.file_server + return connection_server # Connect to host def connect(self, connection=None): @@ -120,13 +156,7 @@ class Peer(object): self.connection = None try: - if self.connection_server: - connection_server = self.connection_server - elif self.site: - connection_server = self.site.connection_server - else: - import main - connection_server = main.file_server + connection_server = self.getConnectionServer() self.connection = connection_server.getConnection(self.ip, self.port, site=self.site, is_tracker_connection=self.is_tracker_connection) if self.connection and self.connection.connected: self.reputation += 1 @@ -183,6 +213,7 @@ class Peer(object): if source in ("tracker", "local"): self.site.peers_recent.appendleft(self) self.time_found = time.time() + self.updateReachable() # Send a command to peer and return response value def request(self, cmd, params={}, stream_to=None): @@ -355,6 +386,8 @@ class Peer(object): # List modified files since the date # Return: {inner_path: modification date,...} def listModified(self, since): + if self.removed: + return False return self.request("listModified", {"since": since, "site": self.site.address}) def updateHashfield(self, force=False): @@ -430,12 +463,11 @@ class Peer(object): # Stop and remove from site def remove(self, reason="Removing"): - self.log("Removing peer...Connection error: %s, Hash failed: %s" % (self.connection_error, self.hash_failed)) - if self.site and self.key in self.site.peers: - del(self.site.peers[self.key]) - - if self.site and self in self.site.peers_recent: - self.site.peers_recent.remove(self) + self.removed = True + self.log("Removing peer with reason: <%s>. Connection error: %s, Hash failed: %s" % (reason, self.connection_error, self.hash_failed)) + if self.site: + self.site.deregisterPeer(self) + self.site = None self.disconnect(reason) @@ -443,6 +475,8 @@ class Peer(object): # On connection error def onConnectionError(self, reason="Unknown"): + if not self.getConnectionServer().isInternetOnline(): + return self.connection_error += 1 if self.site and len(self.site.peers) > 200: limit = 3 @@ -450,7 +484,7 @@ class Peer(object): limit = 6 self.reputation -= 1 if self.connection_error >= limit: # Dead peer - self.remove("Peer connection: %s" % reason) + self.remove("Connection error limit reached: %s. Provided message: %s" % (limit, reason)) # Done working with peer def onWorkerDone(self): diff --git a/src/Site/Site.py b/src/Site/Site.py index 456d02f1..d4f2ad62 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -913,7 +913,7 @@ class Site(object): self.peer_connector.newReq(limit, limit / 2, 10), # some of them... self.peer_connector.newReq(1, 1, 60) # or at least one... ] - self.waitForPeers(reqs) + self.waitForPeers(reqs, update_websocket=False) peers = self.getConnectedPeers() random.shuffle(peers) @@ -1197,6 +1197,15 @@ class Site(object): return peer + # Called from peer.remove to erase links to peer + def deregisterPeer(self, peer): + self.peers.pop(peer.key, None) + try: + self.peers_recent.remove(peer) + except: + pass + self.peer_connector.deregisterPeer(peer) + def announce(self, *args, **kwargs): if self.isServing(): self.announcer.announce(*args, **kwargs) @@ -1295,8 +1304,9 @@ class Site(object): req = self.peer_connector.newReq(0, num) return req - # Wait for peers to ne known and/or connected and send updates to the UI - def waitForPeers(self, reqs): + # Wait for peers to be discovered and/or connected according to reqs + # and send updates to the UI + def waitForPeers(self, reqs, update_websocket=True): if not reqs: return 0 i = 0 @@ -1315,84 +1325,72 @@ class Site(object): waiting_req.waitHeartbeat(timeout=1.0) if i > 0 and nr_connected_peers != waiting_req.nr_connected_peers: nr_connected_peers = waiting_req.nr_connected_peers - self.updateWebsocket(connecting_to_peers=nr_connected_peers) + if update_websocket: + self.updateWebsocket(connecting_to_peers=nr_connected_peers) i += 1 - self.updateWebsocket(connected_to_peers=max(nr_connected_peers, 0)) - if i > 1: - # If we waited some time, pause now for displaying connected_to_peers message in the UI. - # This sleep is solely needed for site status updates on ZeroHello to be more cool-looking. - gevent.sleep(1) + if update_websocket: + self.updateWebsocket(connected_to_peers=max(nr_connected_peers, 0)) + if i > 1: + # If we waited some time, pause now for displaying connected_to_peers message in the UI. + # This sleep is solely needed for site status updates on ZeroHello to be more cool-looking. + gevent.sleep(1) return nr_connected_peers ############################################################################ - # Return: Probably peers verified to be connectable recently + # Return: Peers verified to be connectable recently, or if not enough, other peers as well def getConnectablePeers(self, need_num=5, ignore=[], allow_private=True): peers = list(self.peers.values()) - found = [] + random.shuffle(peers) + connectable_peers = [] + reachable_peers = [] for peer in peers: - if peer.key.endswith(":0"): - continue # Not connectable - if not peer.connection: - continue # No connection - if peer.ip.endswith(".onion") and not self.connection_server.tor_manager.enabled: - continue # Onion not supported if peer.key in ignore: - continue # The requester has this peer - if time.time() - peer.connection.last_recv_time > 60 * 60 * 2: # Last message more than 2 hours ago - peer.connection = None # Cleanup: Dead connection continue if not allow_private and helper.isPrivateIp(peer.ip): continue - found.append(peer) - if len(found) >= need_num: + if peer.isConnectable(): + connectable_peers.append(peer) + elif peer.isReachable(): + reachable_peers.append(peer) + if len(connectable_peers) >= need_num: break # Found requested number of peers - if len(found) < need_num: # Return not that good peers - found += [ - peer for peer in peers - if not peer.key.endswith(":0") and - peer.key not in ignore and - (allow_private or not helper.isPrivateIp(peer.ip)) - ][0:need_num - len(found)] + if len(connectable_peers) < need_num: # Return not that good peers + connectable_peers += reachable_peers[0:need_num - len(connectable_peers)] - return found + return connectable_peers # Return: Recently found peers + def getReachablePeers(self): + return [peer for peer in self.peers.values() if peer.isReachable()] + + # Return: Recently found peers, sorted by reputation. + # If there not enough recently found peers, adds other known peers with highest reputation def getRecentPeers(self, need_num): need_num = int(need_num) - found = list(set(self.peers_recent)) + found = set(self.peers_recent) self.log.debug( "Recent peers %s of %s (need: %s)" % (len(found), len(self.peers), need_num) ) - if len(found) >= need_num or len(found) >= len(self.peers): - return sorted( - found, + if len(found) < need_num and len(found) < len(self.peers): + # Add random peers + peers = self.getReachablePeers() + peers = sorted( + list(peers), key=lambda peer: peer.reputation, reverse=True - )[0:need_num] + ) + while len(found) < need_num and len(peers) > 0: + found.add(peers.pop()) - # Add random peers - need_more = need_num - len(found) - if not self.connection_server.tor_manager.enabled: - peers = [peer for peer in self.peers.values() if not peer.ip.endswith(".onion")] - else: - peers = list(self.peers.values()) - - self.log.debug("getRecentPeers: peers = %s" % peers) - self.log.debug("getRecentPeers: need_more = %s" % need_more) - - peers = peers[0:need_more * 50] - - found_more = sorted( - peers, + return sorted( + list(found), key=lambda peer: peer.reputation, reverse=True - )[0:need_more * 2] - - found += found_more + )[0:need_num] return found[0:need_num] @@ -1400,8 +1398,8 @@ class Site(object): # By default the result may contain peers chosen optimistically: # If the connection is being established and 20 seconds have not yet passed # since the connection start time, those peers are included in the result. - # Set onlyFullyConnected=True for restricting only by fully connected peers. - def getConnectedPeers(self, onlyFullyConnected=False): + # Set only_fully_connected=True for restricting only by fully connected peers. + def getConnectedPeers(self, only_fully_connected=False): back = [] if not self.connection_server: return [] @@ -1413,7 +1411,7 @@ class Site(object): if not connection.connected and time.time() - connection.start_time > 20: # Still not connected after 20s continue - if not connection.connected and onlyFullyConnected: # Only fully connected peers + if not connection.connected and only_fully_connected: # Only fully connected peers continue peer = self.peers.get("%s:%s" % (connection.ip, connection.port)) @@ -1434,7 +1432,9 @@ class Site(object): return removed = 0 - if len(peers) > 1000: + if len(peers) > 10000: + ttl = 60 * 2 + elif len(peers) > 1000: ttl = 60 * 60 * 1 elif len(peers) > 100: ttl = 60 * 60 * 4 @@ -1458,7 +1458,7 @@ class Site(object): self.removeDeadPeers() limit = max(self.getActiveConnectionCountLimit(), self.waitingForConnections()) - connected_peers = self.getConnectedPeers(onlyFullyConnected=True) + connected_peers = self.getConnectedPeers(only_fully_connected=True) need_to_close = len(connected_peers) - limit if need_to_close > 0: @@ -1537,7 +1537,7 @@ class Site(object): return False sent = 0 - connected_peers = self.getConnectedPeers(onlyFullyConnected=True) + connected_peers = self.getConnectedPeers(only_fully_connected=True) for peer in connected_peers: if peer.sendMyHashfield(): sent += 1 @@ -1559,7 +1559,7 @@ class Site(object): s = time.time() queried = 0 - connected_peers = self.getConnectedPeers(onlyFullyConnected=True) + connected_peers = self.getConnectedPeers(only_fully_connected=True) for peer in connected_peers: if peer.time_hashfield: continue diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index 31004d02..1cb0a445 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -311,7 +311,7 @@ class SiteAnnouncer(object): for _ in range(5): if not self.site.isServing(): return - peers = self.site.getConnectedPeers(onlyFullyConnected=True) + peers = self.site.getConnectedPeers(only_fully_connected=True) if len(peers) > 0: break time.sleep(2) diff --git a/src/Site/SiteHelpers.py b/src/Site/SiteHelpers.py index c095ad66..53105f65 100644 --- a/src/Site/SiteHelpers.py +++ b/src/Site/SiteHelpers.py @@ -124,7 +124,7 @@ class PeerConnector(object): self.spawnPeerConnectorController(); def processReqs2(self): - self.nr_connected_peers = len(self.site.getConnectedPeers(onlyFullyConnected=True)) + self.nr_connected_peers = len(self.site.getConnectedPeers(only_fully_connected=True)) self.processReqs(nr_connected_peers=self.nr_connected_peers) # For adding new peers when ConnectorController is working. @@ -140,6 +140,12 @@ class PeerConnector(object): if peer not in self.peers: self.peers.append(peer) + def deregisterPeer(self, peer): + try: + self.peers.remove(peer) + except: + pass + def sleep(self, t): self.site.connection_server.sleep(t) @@ -187,6 +193,8 @@ class PeerConnector(object): self.sleep(10) continue + added = 0 + # try connecting to peers while self.keepGoing() and len(self.peer_connector_workers) < self.peer_connector_worker_limit: if len(self.peers) < 1: @@ -204,13 +212,23 @@ class PeerConnector(object): thread = self.site.spawn(self.peerConnectorWorker, peer) self.peer_connector_workers[peer] = thread thread.link(lambda thread, peer=peer: self.peer_connector_workers.pop(peer, None)) + added += 1 + + if not self.keepGoing(): + break + + if not added: + # Looks like all known peers are either connected or being connected, + # so we weren't able to start connecting any peer in this iteration. + # Waiting for the announcer to discover some peers. + self.sleep(20) # wait for more room in self.peer_connector_workers while self.keepGoing() and len(self.peer_connector_workers) >= self.peer_connector_worker_limit: self.sleep(2) if not self.site.connection_server.isInternetOnline(): - self.sleep(20) + self.sleep(30) self.peers = list() self.peer_connector_controller = None From f484c0a1b8ffc8af6ae7e15a9a3375d1f532f9a8 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 26 Oct 2021 17:40:03 +0700 Subject: [PATCH 102/114] fine-tuning FileServer --- src/File/FileServer.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 9ef3ae9c..f9f31163 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -21,6 +21,12 @@ from Debug import Debug log = logging.getLogger("FileServer") +class FakeThread(object): + def __init__(self): + pass + def ready(self): + return False + @PluginManager.acceptPlugins class FileServer(ConnectionServer): @@ -39,7 +45,7 @@ class FileServer(ConnectionServer): self.active_mode_thread_pool = gevent.pool.Pool(None) self.site_pool = gevent.pool.Pool(None) - self.update_pool = gevent.pool.Pool(5) + self.update_pool = gevent.pool.Pool(10) self.update_start_time = 0 self.update_sites_task_next_nr = 1 @@ -323,9 +329,18 @@ class FileServer(ConnectionServer): def updateSite(self, site, check_files=False, verify_files=False): if not site: return + if verify_files: + mode = 'verify' + elif check_files: + mode = 'check' + else: + mode = 'update' + log.info("running <%s> for %s" % (mode, site.address_short)) site.update2(check_files=check_files, verify_files=verify_files) def spawnUpdateSite(self, site, check_files=False, verify_files=False): + fake_thread = FakeThread() + self.update_threads[site.address] = fake_thread thread = self.update_pool.spawn(self.updateSite, site, check_files=check_files, verify_files=verify_files) self.update_threads[site.address] = thread @@ -395,8 +410,6 @@ class FileServer(ConnectionServer): continue sites_processed += 1 - - log.info("running for %s" % site.address_short) thread = self.spawnUpdateSite(site) if not self.isActiveMode(): @@ -440,8 +453,8 @@ class FileServer(ConnectionServer): def sitesVerificationThread(self): log.info("sitesVerificationThread started") - short_timeout = 10 - long_timeout = 60 + short_timeout = 20 + long_timeout = 120 self.sleep(long_timeout) @@ -477,8 +490,6 @@ class FileServer(ConnectionServer): else: continue - log.info("running <%s> for %s" % (mode, site.address_short)) - thread = self.spawnUpdateSite(site, check_files=check_files, verify_files=verify_files) From 32eb47c482f17455518c909975d86d8f4d14dae6 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Tue, 26 Oct 2021 22:32:28 +0700 Subject: [PATCH 103/114] a small fix in Peer --- src/Peer/Peer.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 28a7220b..aad25110 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -217,6 +217,9 @@ class Peer(object): # Send a command to peer and return response value def request(self, cmd, params={}, stream_to=None): + if self.removed: + return False + if not self.connection or self.connection.closed: self.connect() if not self.connection: @@ -261,6 +264,9 @@ class Peer(object): # Get a file content from peer def getFile(self, site, inner_path, file_size=None, pos_from=0, pos_to=None, streaming=False): + if self.removed: + return False + if file_size and file_size > 5 * 1024 * 1024: max_read_size = 1024 * 1024 else: @@ -315,6 +321,9 @@ class Peer(object): # Send a ping request def ping(self, timeout=10.0, tryes=3): + if self.removed: + return False + response_time = None for retry in range(1, tryes): # Retry 3 times s = time.time() @@ -338,6 +347,9 @@ class Peer(object): # Request peer exchange from peer def pex(self, site=None, need_num=5, request_interval=60*2): + if self.removed: + return False + if not site: site = self.site # If no site defined request peers for this site @@ -391,6 +403,9 @@ class Peer(object): return self.request("listModified", {"since": since, "site": self.site.address}) def updateHashfield(self, force=False): + if self.removed: + return False + # Don't update hashfield again in 5 min if self.time_hashfield and time.time() - self.time_hashfield < 5 * 60 and not force: return False @@ -406,6 +421,9 @@ class Peer(object): # Find peers for hashids # Return: {hash1: ["ip:port", "ip:port",...],...} def findHashIds(self, hash_ids): + if self.removed: + return False + res = self.request("findHashIds", {"site": self.site.address, "hash_ids": hash_ids}) if not res or "error" in res or type(res) is not dict: return False @@ -449,6 +467,9 @@ class Peer(object): return True def publish(self, address, inner_path, body, modified, diffs=[]): + if self.removed: + return False + if len(body) > 10 * 1024 and self.connection and self.connection.handshake.get("rev", 0) >= 4095: # To save bw we don't push big content.json to peers body = b"" @@ -467,7 +488,9 @@ class Peer(object): self.log("Removing peer with reason: <%s>. Connection error: %s, Hash failed: %s" % (reason, self.connection_error, self.hash_failed)) if self.site: self.site.deregisterPeer(self) - self.site = None + # No way: self.site = None + # We don't assign None to self.site here because it leads to random exceptions in various threads, + # that hold references to the peer and still believe it belongs to the site. self.disconnect(reason) From ef69dcd331b4123c3bec267b23c26b0b4807afb1 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 27 Oct 2021 01:06:12 +0700 Subject: [PATCH 104/114] Implement Send Back LRU cache to reduce useless network transfers --- src/Config.py | 3 +++ src/Site/Site.py | 28 +++++++++++++++++++++++----- src/Site/SiteManager.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/Config.py b/src/Config.py index 67f65f7b..26fbafb2 100644 --- a/src/Config.py +++ b/src/Config.py @@ -268,6 +268,9 @@ class Config(object): self.parser.add_argument('--global_connected_limit', help='Max connections', default=512, type=int, metavar='global_connected_limit') self.parser.add_argument('--workers', help='Download workers per site', default=5, type=int, metavar='workers') + self.parser.add_argument('--send_back_lru_size', help='Size of the send back LRU cache', default=5000, type=int, metavar='send_back_lru_size') + self.parser.add_argument('--send_back_limit', help='Send no more than so many files at once back to peer, when we discovered that the peer held older file versions', default=3, type=int, metavar='send_back_limit') + self.parser.add_argument('--expose_no_ownership', help='By default, ZeroNet tries checking updates for own sites more frequently. This can be used by a third party for revealing the network addresses of a site owner. If this option is enabled, ZeroNet performs the checks in the same way for any sites.', type='bool', choices=[True, False], default=False) self.parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') diff --git a/src/Site/Site.py b/src/Site/Site.py index d4f2ad62..73bb01dc 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -212,7 +212,6 @@ class Site(object): self.peer_connector = SiteHelpers.PeerConnector(self) # Connect more peers in background by request self.persistent_peer_req = None # The persistent peer requirement, managed by maintenance handler - if not self.settings.get("wrapper_key"): # To auth websocket permissions self.settings["wrapper_key"] = CryptHash.random() self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"]) @@ -308,6 +307,12 @@ class Site(object): thread = self.greenlet_manager.spawnLater(*args, **kwargs) return thread + def checkSendBackLRU(self, peer, inner_path, remote_modified): + return SiteManager.site_manager.checkSendBackLRU(self, peer, inner_path, remote_modified) + + def addToSendBackLRU(self, peer, inner_path, modified): + return SiteManager.site_manager.addToSendBackLRU(self, peer, inner_path, modified) + def getSettingsCache(self): back = {} back["bad_files"] = self.bad_files @@ -619,7 +624,8 @@ class Site(object): queried.append(peer) modified_contents = [] send_back = [] - send_back_limit = 5 + send_back_limit = config.send_back_limit + send_back_skipped = 0 my_modified = self.content_manager.listModified(since) num_old_files = 0 for inner_path, modified in res["modified_files"].items(): # Check if the peer has newer files than we @@ -631,7 +637,10 @@ class Site(object): modified_contents.append(inner_path) self.bad_files[inner_path] = self.bad_files.get(inner_path, 1) if has_older: - send_back.append(inner_path) + if self.checkSendBackLRU(peer, inner_path, modified): + send_back_skipped += 1 + else: + send_back.append(inner_path) if modified_contents: self.log.info("CheckModifications: %s new modified files from %s" % (len(modified_contents), peer)) @@ -653,7 +662,10 @@ class Site(object): self.log.info("CheckModifications: %s: %s < %s" % ( inner_path, res["modified_files"][inner_path], my_modified.get(inner_path, 0) )) - self.spawn(self.publisher, inner_path, [peer], [], 1) + self.spawn(self.publisher, inner_path, [peer], [], 1, save_to_send_back_lru=True) + + if send_back_skipped: + self.log.info("CheckModifications: %s has older versions of %s files, skipped according to send back LRU" % (peer, send_back_skipped)) self.log.debug("CheckModifications: Waiting for %s pooledDownloadContent" % len(threads)) gevent.joinall(threads) @@ -842,7 +854,7 @@ class Site(object): gevent.joinall(content_threads) # Publish worker - def publisher(self, inner_path, peers, published, limit, diffs={}, event_done=None, cb_progress=None, max_retries=2): + def publisher(self, inner_path, peers, published, limit, diffs={}, event_done=None, cb_progress=None, max_retries=2, save_to_send_back_lru=False): file_size = self.storage.getSize(inner_path) content_json_modified = self.content_manager.contents[inner_path]["modified"] body = self.storage.read(inner_path) @@ -877,6 +889,12 @@ class Site(object): self.log.error("Publish error: %s" % Debug.formatException(err)) result = {"exception": Debug.formatException(err)} + # We add to the send back lru not only on success, but also on errors. + # Some peers returns None. (Why?) + # Anyway, we tried our best in delivering possibly lost updates. + if save_to_send_back_lru: + self.addToSendBackLRU(peer, inner_path, content_json_modified) + if result and "ok" in result: published.append(peer) if cb_progress and len(published) <= limit: diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 1b065e93..035e9279 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -4,6 +4,7 @@ import re import os import time import atexit +import collections import gevent @@ -27,6 +28,21 @@ class SiteManager(object): gevent.spawn(self.saveTimer) atexit.register(lambda: self.save(recalculate_size=True)) + # ZeroNet has a bug of desyncing between: + # * time sent in a response of listModified + # and + # * time checked on receiving a file. + # This leads to the following scenario: + # * Request listModified. + # * Detect that the remote peer missing an update + # * Send a newer version of the file back to the peer. + # * The peer responses "ok: File not changed" + # ..... + # * Request listModified the next time and do all the same again. + # So we keep the list of sent back entries to prevent sending multiple useless updates: + # "{site.address} - {peer.key} - {inner_path}" -> mtime + self.send_back_lru = collections.OrderedDict() + # Load all sites from data/sites.json @util.Noparallel() def load(self, cleanup=True, startup=False): @@ -220,6 +236,23 @@ class SiteManager(object): self.load(startup=True) return self.sites + # Return False if we never sent to + # or if the file that was sent was older than + # so that send back logic is suppressed for . + # True if can be sent back to . + def checkSendBackLRU(self, site, peer, inner_path, remote_modified): + key = site.address + ' - ' + peer.key + ' - ' + inner_path + sent_modified = self.send_back_lru.get(key, 0) + return remote_modified < sent_modified + + def addToSendBackLRU(self, site, peer, inner_path, modified): + key = site.address + ' - ' + peer.key + ' - ' + inner_path + if self.send_back_lru.get(key, None) is None: + self.send_back_lru[key] = modified + while len(self.send_back_lru) > config.send_back_lru_size: + self.send_back_lru.popitem(last=False) + else: + self.send_back_lru.move_to_end(key, last=True) site_manager = SiteManager() # Singletone From 77e0bb3650ca3e79c0ae14e8de3840e7accbbaaa Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 27 Oct 2021 18:54:58 +0700 Subject: [PATCH 105/114] PeerDb plugin: save and restore fields time_response and connection_error --- plugins/PeerDb/PeerDbPlugin.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/plugins/PeerDb/PeerDbPlugin.py b/plugins/PeerDb/PeerDbPlugin.py index a66b81cf..aea11fbb 100644 --- a/plugins/PeerDb/PeerDbPlugin.py +++ b/plugins/PeerDb/PeerDbPlugin.py @@ -24,12 +24,14 @@ class ContentDbPlugin(object): ["hashfield", "BLOB"], ["reputation", "INTEGER NOT NULL"], ["time_added", "INTEGER NOT NULL"], - ["time_found", "INTEGER NOT NULL"] + ["time_found", "INTEGER NOT NULL"], + ["time_response", "INTEGER NOT NULL"], + ["connection_error", "INTEGER NOT NULL"] ], "indexes": [ "CREATE UNIQUE INDEX peer_key ON peer (site_id, address, port)" ], - "schema_changed": 2 + "schema_changed": 3 } return schema @@ -49,9 +51,15 @@ class ContentDbPlugin(object): num_hashfield += 1 peer.time_added = row["time_added"] peer.time_found = row["time_found"] - peer.reputation = row["reputation"] + peer.time_found = row["time_found"] + peer.time_response = row["time_response"] + peer.connection_error = row["connection_error"] if row["address"].endswith(".onion"): - peer.reputation = peer.reputation / 2 - 1 # Onion peers less likely working + # Onion peers less likely working + if peer.reputation > 0: + peer.reputation = peer.reputation / 2 + else: + peer.reputation -= 1 num += 1 if num_hashfield: site.content_manager.has_optional_files = True @@ -65,7 +73,7 @@ class ContentDbPlugin(object): hashfield = sqlite3.Binary(peer.hashfield.tobytes()) else: hashfield = "" - yield (site_id, address, port, hashfield, peer.reputation, int(peer.time_added), int(peer.time_found)) + yield (site_id, address, port, hashfield, peer.reputation, int(peer.time_added), int(peer.time_found), int(peer.time_response), int(peer.connection_error)) def savePeers(self, site, spawn=False): if spawn: @@ -80,7 +88,7 @@ class ContentDbPlugin(object): try: cur.execute("DELETE FROM peer WHERE site_id = :site_id", {"site_id": site_id}) cur.executemany( - "INSERT INTO peer (site_id, address, port, hashfield, reputation, time_added, time_found) VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO peer (site_id, address, port, hashfield, reputation, time_added, time_found, time_response, connection_error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", self.iteratePeers(site) ) except Exception as err: From 168c436b735b1d0aa724907c7fc5fdbf0ed28413 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 27 Oct 2021 20:57:44 +0700 Subject: [PATCH 106/114] Add new configuration variables and temporarily disable Site.persistent_peer_req New configuration options: site_announce_interval_min site_announce_interval_max site_peer_check_interval_min site_peer_check_interval_max site_update_check_interval_min site_update_check_interval_max site_connectable_peer_count_max site_connectable_peer_count_min Site.persistent_peer_req is temporarily disabled since it makes excessive pressure on the network when working over TOR and needs some reworking. --- src/Config.py | 16 +++++++-- src/Site/Site.py | 78 ++++++++++++++++++++++++++++++++++++----- src/Site/SiteHelpers.py | 1 + 3 files changed, 85 insertions(+), 10 deletions(-) diff --git a/src/Config.py b/src/Config.py index 26fbafb2..15a0c87f 100644 --- a/src/Config.py +++ b/src/Config.py @@ -264,10 +264,22 @@ class Config(object): self.parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, type=int, metavar='limit') self.parser.add_argument('--file_size_limit', help='Maximum per file size limit in MB', default=10, type=int, metavar='limit') - self.parser.add_argument('--connected_limit', help='Max connected peer per site', default=10, type=int, metavar='connected_limit') - self.parser.add_argument('--global_connected_limit', help='Max connections', default=512, type=int, metavar='global_connected_limit') + self.parser.add_argument('--connected_limit', help='Max number of connected peers per site. Soft limit.', default=10, type=int, metavar='connected_limit') + self.parser.add_argument('--global_connected_limit', help='Max number of connections. Soft limit.', default=512, type=int, metavar='global_connected_limit') self.parser.add_argument('--workers', help='Download workers per site', default=5, type=int, metavar='workers') + self.parser.add_argument('--site_announce_interval_min', help='Site announce interval for the most active sites, in minutes.', default=4, type=int, metavar='site_announce_interval_min') + self.parser.add_argument('--site_announce_interval_max', help='Site announce interval for inactive sites, in minutes.', default=30, type=int, metavar='site_announce_interval_max') + + self.parser.add_argument('--site_peer_check_interval_min', help='Connectable peers check interval for the most active sites, in minutes.', default=5, type=int, metavar='site_peer_check_interval_min') + self.parser.add_argument('--site_peer_check_interval_max', help='Connectable peers check interval for inactive sites, in minutes.', default=20, type=int, metavar='site_peer_check_interval_max') + + self.parser.add_argument('--site_update_check_interval_min', help='Site update check interval for the most active sites, in minutes.', default=5, type=int, metavar='site_update_check_interval_min') + self.parser.add_argument('--site_update_check_interval_max', help='Site update check interval for inactive sites, in minutes.', default=45, type=int, metavar='site_update_check_interval_max') + + self.parser.add_argument('--site_connectable_peer_count_max', help='Search for as many connectable peers for the most active sites', default=10, type=int, metavar='site_connectable_peer_count_max') + self.parser.add_argument('--site_connectable_peer_count_min', help='Search for as many connectable peers for inactive sites', default=2, type=int, metavar='site_connectable_peer_count_min') + self.parser.add_argument('--send_back_lru_size', help='Size of the send back LRU cache', default=5000, type=int, metavar='send_back_lru_size') self.parser.add_argument('--send_back_limit', help='Send no more than so many files at once back to peer, when we discovered that the peer held older file versions', default=3, type=int, metavar='send_back_limit') diff --git a/src/Site/Site.py b/src/Site/Site.py index 73bb01dc..0cea4733 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -30,6 +30,9 @@ from .SiteAnnouncer import SiteAnnouncer from . import SiteManager from . import SiteHelpers +def lerp(val_min, val_max, scale): + return scale * (val_max - val_min) + val_min + class ScaledTimeoutHandler: def __init__(self, val_min, val_max, handler=None, scaler=None): self.val_min = val_min @@ -40,7 +43,7 @@ class ScaledTimeoutHandler: self.log = logging.getLogger("ScaledTimeoutHandler") def isExpired(self, scale): - interval = scale * (self.val_max - self.val_min) + self.val_min + interval = lerp(self.val_min, self.val_max, scale) expired_at = self.timestamp + interval now = time.time() expired = (now > expired_at) @@ -158,10 +161,19 @@ class Site(object): self.addEventListeners() self.periodic_maintenance_handlers = [ - ScaledTimeoutHandler(60 * 30, 60 * 2, + ScaledTimeoutHandler( + config.site_announce_interval_max * 60, + config.site_announce_interval_min * 60, handler=self.periodicMaintenanceHandler_announce, scaler=self.getAnnounceRating), - ScaledTimeoutHandler(60 * 30, 60 * 5, + ScaledTimeoutHandler( + config.site_peer_check_interval_max * 60, + config.site_peer_check_interval_min * 60, + handler=self.periodicMaintenanceHandler_peer_check, + scaler=self.getAnnounceRating), + ScaledTimeoutHandler( + config.site_update_check_interval_max * 60, + config.site_update_check_interval_min * 60, handler=self.periodicMaintenanceHandler_general, scaler=self.getActivityRating) ] @@ -1279,6 +1291,16 @@ class Site(object): v = [activity_rating, peer_rating, tracker_rating] return sum(v) / float(len(v)) + def getPreferableConnectablePeerCount(self): + if not self.isServing(): + return 0 + + count = lerp( + config.site_connectable_peer_count_min, + config.site_connectable_peer_count_max, + self.getActivityRating(force_safe=True)) + return count + # The engine tries to maintain the number of active connections: # >= getPreferableActiveConnectionCount() # and @@ -1496,6 +1518,33 @@ class Site(object): self.log.debug("Connected: %s, Need to close: %s, Closed: %s" % ( len(connected_peers), need_to_close, closed)) + def lookForConnectablePeers(self): + num_tries = 2 + need_connectable_peers = self.getPreferableConnectablePeerCount() + + connectable_peers = 0 + reachable_peers = [] + + for peer in list(self.peers.values()): + if peer.isConnected() or peer.isConnectable(): + connectable_peers += 1 + elif peer.isReachable(): + reachable_peers.append(peer) + if connectable_peers >= need_connectable_peers: + return True + + random.shuffle(reachable_peers) + + for peer in reachable_peers: + if peer.isConnected() or peer.isConnectable() or peer.removed: + continue + peer.ping() + if peer.isConnected(): + peer.pex() + num_tries -= 1 + if num_tries < 1: + break + @util.Noparallel(queue=True) def runPeriodicMaintenance(self, startup=False, force=False): if not self.isServing(): @@ -1519,11 +1568,8 @@ class Site(object): self.log.debug("periodicMaintenanceHandler_general: startup=%s, force=%s" % (startup, force)) - if not startup: - self.cleanupPeers() - - self.persistent_peer_req = self.needConnections(update_site_on_reconnect=True) - self.persistent_peer_req.result_connected.wait(timeout=2.0) + #self.persistent_peer_req = self.needConnections(update_site_on_reconnect=True) + #self.persistent_peer_req.result_connected.wait(timeout=2.0) #self.announcer.announcePex() @@ -1533,6 +1579,22 @@ class Site(object): return True + def periodicMaintenanceHandler_peer_check(self, startup=False, force=False): + if not self.isServing(): + return False + + if not self.peers: + return False + + self.log.debug("periodicMaintenanceHandler_peer_check: startup=%s, force=%s" % (startup, force)) + + if not startup: + self.cleanupPeers() + + self.lookForConnectablePeers() + + return True + def periodicMaintenanceHandler_announce(self, startup=False, force=False): if not self.isServing(): return False diff --git a/src/Site/SiteHelpers.py b/src/Site/SiteHelpers.py index 53105f65..90a298cf 100644 --- a/src/Site/SiteHelpers.py +++ b/src/Site/SiteHelpers.py @@ -156,6 +156,7 @@ class PeerConnector(object): if not peer.isConnected(): peer.connect() if peer.isConnected(): + peer.ping() self.processReqs2() def peerConnectorController(self): From d32d9f781b81555d64e4b29fdafd497fbf2e31c9 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Nov 2021 11:48:02 +0700 Subject: [PATCH 107/114] Move getIpType() from helper to ConnectionServer --- .../AnnounceBitTorrentPlugin.py | 4 +- plugins/TrackerZero/TrackerZeroPlugin.py | 3 +- .../BootstrapperPlugin.py | 2 +- .../Test/TestBootstrapper.py | 6 +- src/Connection/Connection.py | 6 +- src/Connection/ConnectionServer.py | 36 ++++++++++- src/File/FileRequest.py | 4 +- src/File/FileServer.py | 8 +-- src/Peer/Peer.py | 63 ++++++++++++------- src/Site/SiteAnnouncer.py | 20 +++--- src/Site/SiteManager.py | 1 + src/util/helper.py | 3 +- 12 files changed, 109 insertions(+), 47 deletions(-) diff --git a/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py b/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py index fab7bb1f..734070dd 100644 --- a/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py +++ b/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py @@ -52,7 +52,7 @@ class SiteAnnouncerPlugin(object): ip, port = tracker_address.split("/")[0].split(":") tracker = UdpTrackerClient(ip, int(port)) - if helper.getIpType(ip) in self.getOpenedServiceTypes(): + if self.connection_server.getIpType(ip) in self.getOpenedServiceTypes(): tracker.peer_port = self.fileserver_port else: tracker.peer_port = 0 @@ -101,7 +101,7 @@ class SiteAnnouncerPlugin(object): def announceTrackerHttp(self, tracker_address, mode="start", num_want=10, protocol="http"): tracker_ip, tracker_port = tracker_address.rsplit(":", 1) - if helper.getIpType(tracker_ip) in self.getOpenedServiceTypes(): + if self.connection_server.getIpType(tracker_ip) in self.getOpenedServiceTypes(): port = self.fileserver_port else: port = 1 diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py index e90f085d..a59bc309 100644 --- a/plugins/TrackerZero/TrackerZeroPlugin.py +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -122,7 +122,8 @@ class TrackerZero(object): time_onion_check = time.time() - s - ip_type = helper.getIpType(file_request.connection.ip) + connection_server = file_request.server + ip_type = connection_server.getIpType(file_request.connection.ip) if ip_type == "onion" or file_request.connection.ip in config.ip_local: is_port_open = False diff --git a/plugins/disabled-Bootstrapper/BootstrapperPlugin.py b/plugins/disabled-Bootstrapper/BootstrapperPlugin.py index 59e7af7b..5ddc36b6 100644 --- a/plugins/disabled-Bootstrapper/BootstrapperPlugin.py +++ b/plugins/disabled-Bootstrapper/BootstrapperPlugin.py @@ -49,7 +49,7 @@ class FileRequestPlugin(object): time_onion_check = time.time() - s - ip_type = helper.getIpType(self.connection.ip) + ip_type = self.server.getIpType(self.connection.ip) if ip_type == "onion" or self.connection.ip in config.ip_local: is_port_open = False diff --git a/plugins/disabled-Bootstrapper/Test/TestBootstrapper.py b/plugins/disabled-Bootstrapper/Test/TestBootstrapper.py index 69bdc54c..198cd022 100644 --- a/plugins/disabled-Bootstrapper/Test/TestBootstrapper.py +++ b/plugins/disabled-Bootstrapper/Test/TestBootstrapper.py @@ -28,7 +28,7 @@ def bootstrapper_db(request): @pytest.mark.usefixtures("resetSettings") class TestBootstrapper: def testHashCache(self, file_server, bootstrapper_db): - ip_type = helper.getIpType(file_server.ip) + ip_type = file_server.getIpType(file_server.ip) peer = Peer(file_server.ip, 1544, connection_server=file_server) hash1 = hashlib.sha256(b"site1").digest() hash2 = hashlib.sha256(b"site2").digest() @@ -50,7 +50,7 @@ class TestBootstrapper: def testBootstrapperDb(self, file_server, bootstrapper_db): - ip_type = helper.getIpType(file_server.ip) + ip_type = file_server.getIpType(file_server.ip) peer = Peer(file_server.ip, 1544, connection_server=file_server) hash1 = hashlib.sha256(b"site1").digest() hash2 = hashlib.sha256(b"site2").digest() @@ -111,7 +111,7 @@ class TestBootstrapper: def testPassive(self, file_server, bootstrapper_db): peer = Peer(file_server.ip, 1544, connection_server=file_server) - ip_type = helper.getIpType(file_server.ip) + ip_type = file_server.getIpType(file_server.ip) hash1 = hashlib.sha256(b"hash1").digest() bootstrapper_db.peerAnnounce(ip_type, address=None, port=15441, hashes=[hash1]) diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 40519b7f..c147ee35 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -23,6 +23,7 @@ class Connection(object): ) def __init__(self, server, ip, port, sock=None, target_onion=None, is_tracker_connection=False): + self.server = server self.sock = sock self.cert_pin = None if "#" in ip: @@ -42,7 +43,6 @@ class Connection(object): self.is_private_ip = False self.is_tracker_connection = is_tracker_connection - self.server = server self.unpacker = None # Stream incoming socket messages here self.unpacker_bytes = 0 # How many bytes the unpacker received self.req_id = 0 # Last request id @@ -81,11 +81,11 @@ class Connection(object): def setIp(self, ip): self.ip = ip - self.ip_type = helper.getIpType(ip) + self.ip_type = self.server.getIpType(ip) self.updateName() def createSocket(self): - if helper.getIpType(self.ip) == "ipv6" and not hasattr(socket, "socket_noproxy"): + if self.server.getIpType(self.ip) == "ipv6" and not hasattr(socket, "socket_noproxy"): # Create IPv6 connection as IPv4 when using proxy return socket.socket(socket.AF_INET6, socket.SOCK_STREAM) else: diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 16834ff5..9f24e377 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -1,4 +1,5 @@ import logging +import re import time import sys import socket @@ -253,7 +254,7 @@ class ConnectionServer(object): pass def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None, is_tracker_connection=False): - ip_type = helper.getIpType(ip) + ip_type = self.getIpType(ip) has_per_site_onion = (ip.endswith(".onion") or self.port_opened.get(ip_type, None) == False) and self.tor_manager.start_onions and site if has_per_site_onion: # Site-unique connection for Tor if ip.endswith(".onion"): @@ -520,16 +521,47 @@ class ConnectionServer(object): median = (corrections[mid - 1] + corrections[mid] + corrections[mid + 1]) / 3 return median + + ############################################################################ + + # Methods for handling network address types + # (ipv4, ipv6, onion etc... more to be implemented by plugins) + # + # All the functions handling network address types have "Ip" in the name. + # So it was in the initial codebase, and I keep the naming, since I couldn't + # think of a better option. + # "IP" is short and quite clear and lets you understand that a variable + # contains a peer address or other transport-level address and not + # an address of ZeroNet site. + # + + # Returns type of the given network address. + # Since: 0.8.0 + # Replaces helper.getIpType() in order to be extensible by plugins. + def getIpType(self, ip): + if ip.endswith(".onion"): + return "onion" + elif ":" in ip: + return "ipv6" + elif re.match(r"[0-9\.]+$", ip): + return "ipv4" + else: + return "unknown" + # Checks if a network address can be reachable in the current configuration # and returs a string describing why it cannot. # If the network address can be reachable, returns False. + # Since: 0.8.0 def getIpUnreachability(self, ip): - ip_type = helper.getIpType(ip) + ip_type = self.getIpType(ip) if ip_type == 'onion' and not self.tor_manager.enabled: return "Can't connect to onion addresses, no Tor controller present" if config.tor == "always" and helper.isPrivateIp(ip) and ip not in config.ip_local: return "Can't connect to local IPs in Tor: always mode" return False + # Returns True if ConnctionServer has means for establishing outgoing + # connections to the given address. + # Since: 0.8.0 def isIpReachable(self, ip): return self.getIpUnreachability(ip) == False diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index f7249d81..8a16e591 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -376,7 +376,7 @@ class FileRequest(object): for hash_id, peers in found.items(): for peer in peers: - ip_type = helper.getIpType(peer.ip) + ip_type = self.server.getIpType(peer.ip) if len(back[ip_type][hash_id]) < 20: back[ip_type][hash_id].append(peer.packMyAddress()) return back @@ -430,7 +430,7 @@ class FileRequest(object): # Check requested port of the other peer def actionCheckport(self, params): - if helper.getIpType(self.connection.ip) == "ipv6": + if self.server.getIpType(self.connection.ip) == "ipv6": sock_address = (self.connection.ip, params["port"], 0, 0) else: sock_address = (self.connection.ip, params["port"]) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index f9f31163..ac4b8c55 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -57,7 +57,7 @@ class FileServer(ConnectionServer): self.supported_ip_types = ["ipv4"] # Outgoing ip_type support - if helper.getIpType(ip) == "ipv6" or self.isIpv6Supported(): + if self.getIpType(ip) == "ipv6" or self.isIpv6Supported(): self.supported_ip_types.append("ipv6") if ip_type == "ipv6" or (ip_type == "dual" and "ipv6" in self.supported_ip_types): @@ -217,7 +217,7 @@ class FileServer(ConnectionServer): for ip_external in config.ip_external: SiteManager.peer_blacklist.append((ip_external, self.port)) # Add myself to peer blacklist - ip_external_types = set([helper.getIpType(ip) for ip in config.ip_external]) + ip_external_types = set([self.getIpType(ip) for ip in config.ip_external]) res = { "ipv4": "ipv4" in ip_external_types, "ipv6": "ipv6" in ip_external_types @@ -245,7 +245,7 @@ class FileServer(ConnectionServer): res_ipv6 = {"ip": None, "opened": None} else: res_ipv6 = res_ipv6_thread.get() - if res_ipv6["opened"] and not helper.getIpType(res_ipv6["ip"]) == "ipv6": + if res_ipv6["opened"] and not self.getIpType(res_ipv6["ip"]) == "ipv6": log.info("Invalid IPv6 address from port check: %s" % res_ipv6["ip"]) res_ipv6["opened"] = False @@ -266,7 +266,7 @@ class FileServer(ConnectionServer): for ip in interface_ips: if not helper.isPrivateIp(ip) and ip not in self.ip_external_list: self.ip_external_list.append(ip) - res[helper.getIpType(ip)] = True # We have opened port if we have external ip + res[self.getIpType(ip)] = True # We have opened port if we have external ip SiteManager.peer_blacklist.append((ip, self.port)) log.debug("External ip found on interfaces: %s" % ip) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index aad25110..6e4863e0 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -32,7 +32,7 @@ class Peer(object): self.site = site self.key = "%s:%s" % (ip, port) - self.ip_type = helper.getIpType(ip) + self.ip_type = None self.removed = False @@ -53,7 +53,7 @@ class Peer(object): self.reputation = 0 # More likely to connect if larger self.last_content_json_update = 0.0 # Modify date of last received content.json self.protected = 0 - self.reachable = False + self.reachable = None self.connection_error = 0 # Series of connection error self.hash_failed = 0 # Number of bad files from peer @@ -62,15 +62,14 @@ class Peer(object): self.protectedRequests = ["getFile", "streamFile", "update", "listModified"] - self.updateReachable() - def __getattr__(self, key): if key == "hashfield": self.has_hashfield = True self.hashfield = PeerHashfield() return self.hashfield else: - return getattr(self, key) + # Raise appropriately formatted attribute error + return object.__getattribute__(self, key) def log(self, text, log_level = None): if log_level is None: @@ -98,26 +97,18 @@ class Peer(object): self.protected = 0 return self.protected > 0 + def isTtlExpired(self, ttl): + last_activity = max(self.time_found, self.time_response) + return (time.time() - last_activity) > ttl + + # Since 0.8.0 def isConnected(self): if self.connection and not self.connection.connected: self.connection = None return self.connection and self.connection.connected - def isTtlExpired(self, ttl): - last_activity = max(self.time_found, self.time_response) - return (time.time() - last_activity) > ttl - - def isReachable(self): - return self.reachable - - def updateReachable(self): - connection_server = self.getConnectionServer() - if not self.port: - self.reachable = False - else: - self.reachable = connection_server.isIpReachable(self.ip) - # Peer proved to to be connectable recently + # Since 0.8.0 def isConnectable(self): if self.connection_error >= 1: # The last connection attempt failed return False @@ -125,6 +116,36 @@ class Peer(object): return False return self.isReachable() + # Since 0.8.0 + def isReachable(self): + if self.reachable is None: + self.updateCachedState() + return self.reachable + + # Since 0.8.0 + def getIpType(self): + if not self.ip_type: + self.updateCachedState() + return self.ip_type + + # We cache some ConnectionServer-related state for better performance. + # This kind of state currently doesn't change during a program session, + # and it's safe to read and cache it just once. But future versions + # may bring more pieces of dynamic configuration. So we update the state + # on each peer.found(). + def updateCachedState(self): + connection_server = self.getConnectionServer() + if not self.port or self.port == 1: # Port 1 considered as "no open port" + self.reachable = False + else: + self.reachable = connection_server.isIpReachable(self.ip) + self.ip_type = connection_server.getIpType(self.ip) + + + # FIXME: + # This should probably be changed. + # When creating a peer object, the caller must provide either `connection_server`, + # or `site`, so Peer object is able to use `site.connection_server`. def getConnectionServer(self): if self.connection_server: connection_server = self.connection_server @@ -179,7 +200,7 @@ class Peer(object): if self.connection and self.connection.connected: # We have connection to peer return self.connection else: # Try to find from other sites connections - self.connection = self.site.connection_server.getConnection(self.ip, self.port, create=False, site=self.site) + self.connection = self.getConnectionServer().getConnection(self.ip, self.port, create=False, site=self.site) if self.connection: self.connection.sites += 1 return self.connection @@ -213,7 +234,7 @@ class Peer(object): if source in ("tracker", "local"): self.site.peers_recent.appendleft(self) self.time_found = time.time() - self.updateReachable() + self.updateCachedState() # Send a command to peer and return response value def request(self, cmd, params={}, stream_to=None): diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index 1cb0a445..1baf39af 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -35,19 +35,25 @@ class SiteAnnouncer(object): self.time_last_announce = 0 self.supported_tracker_count = 0 + # Returns connection_server rela + # Since 0.8.0 + @property + def connection_server(self): + return self.site.connection_server + def getTrackers(self): return config.trackers def getSupportedTrackers(self): trackers = self.getTrackers() - if not self.site.connection_server.tor_manager.enabled: + if not self.connection_server.tor_manager.enabled: trackers = [tracker for tracker in trackers if ".onion" not in tracker] trackers = [tracker for tracker in trackers if self.getAddressParts(tracker)] # Remove trackers with unknown address - if "ipv6" not in self.site.connection_server.supported_ip_types: - trackers = [tracker for tracker in trackers if helper.getIpType(self.getAddressParts(tracker)["ip"]) != "ipv6"] + if "ipv6" not in self.connection_server.supported_ip_types: + trackers = [tracker for tracker in trackers if self.connection_server.getIpType(self.getAddressParts(tracker)["ip"]) != "ipv6"] return trackers @@ -118,10 +124,10 @@ class SiteAnnouncer(object): back = [] # Type of addresses they can reach me if config.trackers_proxy == "disable" and config.tor != "always": - for ip_type, opened in list(self.site.connection_server.port_opened.items()): + for ip_type, opened in list(self.connection_server.port_opened.items()): if opened: back.append(ip_type) - if self.site.connection_server.tor_manager.start_onions: + if self.connection_server.tor_manager.start_onions: back.append("onion") return back @@ -204,11 +210,11 @@ class SiteAnnouncer(object): self.stats[tracker]["time_status"] = time.time() self.stats[tracker]["last_error"] = str(error) self.stats[tracker]["time_last_error"] = time.time() - if self.site.connection_server.has_internet: + if self.connection_server.has_internet: self.stats[tracker]["num_error"] += 1 self.stats[tracker]["num_request"] += 1 global_stats[tracker]["num_request"] += 1 - if self.site.connection_server.has_internet: + if self.connection_server.has_internet: global_stats[tracker]["num_error"] += 1 self.updateWebsocket(tracker="error") return False diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 035e9279..8175a1f5 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -172,6 +172,7 @@ class SiteManager(object): return self.resolveDomain(domain) # Checks if the address is blocked. To be implemented in content filter plugins. + # Since 0.8.0 def isAddressBlocked(self, address): return False diff --git a/src/util/helper.py b/src/util/helper.py index 61455b08..f44bcfce 100644 --- a/src/util/helper.py +++ b/src/util/helper.py @@ -290,7 +290,8 @@ local_ip_pattern = re.compile(r"^127\.|192\.168\.|10\.|172\.1[6-9]\.|172\.2[0-9] def isPrivateIp(ip): return local_ip_pattern.match(ip) - +# XXX: Deprecated. Use ConnectionServer.getIpType() instead. +# To be removed in 0.9.0 def getIpType(ip): if ip.endswith(".onion"): return "onion" From 18630435051d492b7a953c461392cab0dd7576ef Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Nov 2021 12:59:49 +0700 Subject: [PATCH 108/114] Update ChangeLog-0.8.0.md --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 34 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md index a2bba51b..568b015d 100644 --- a/ZNE-ChangeLog/ChangeLog-0.8.0.md +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -1,14 +1,17 @@ ## Core -**Network:** +**Support for Onion v3:** -* Added support of Onion v3 addresses. Thanks to @anonymoose and @zeroseed. -* Added a few Onion v3 tracker addresses. +* Add initial support of Onion v3 addresses. Thanks to @anonymoose and @zeroseed. +* Add a few Onion v3 tracker addresses. + +**Network scalabily and performance issuses:** + +* The connection handling code had several bugs that were hidden by silently ignored exceptions. These were fixed, but some new ones might be introduced. * Reworked the algorithm of checking zite updates on startup / after the network outages / periodically. ZeroNet tries not to spam too many update queries at once in order to prevent network overload. (Which especially the issue when running over Tor.) At the same time, it tries to keep balance between frequent checks for frequently updating zites and ensuring that all the zites are checked in some reasonable time interval. Tests show that the full check cycle for a peer that hosts 800+ zites and is connected over Tor can take up to several hours. We cannot significantly reduce this time, since the Tor throughput is the bottleneck. Running more checks at once just results in more connections to fail. The other bottleneck is the HDD throughput. Increasing the parallelization doesn't help in this case as well. So the implemented solution do actually **decreases** the parallelization. * Improved the Internet outage detection and the recovery procedures after the Internet be back. ZeroNet "steps back" and schedules rechecking zites that were checked shortly before the Internet connection get lost. The network outage detection normally has some lag, so the recently checked zites are better to be checked again. * When the network is down, reduce the frequency of connection attempts to prevent overloading Tor with hanged connections. -* The connection handling code had several bugs that were hidden by silently ignored exceptions. These were fixed, but some new ones might be introduced. * For each zite the activity rate is calculated based on the last modification time. The milestone values are 1 hour, 5 hours, 24 hours, 3 days and 7 days. The activity rate is used to scale frequency of various maintenance routines, including update checks, reannounces, dead connection checks etc. * The activity rate is also used to calculate the minimum preferred number of active connections per each zite. * The reannounce frequency is adjusted dynamically based on: @@ -17,15 +20,28 @@ * Tracker count. More trackers ==> frequent announces to iterate over more trackers. * For owned zites, the activity rate doesn't drop below 0.6 to force more frequent checks. This, however, can be used to investigate which peer belongs to the zite owner. A new commnd line option `--expose_no_ownership` is introduced to disable that behavior. * When checking for updates, ZeroNet normally asks other peers for new data since the previous update. This however can result in losing some updates in specific conditions. To overcome this, ZeroNet now asks for the full site listing on every Nth update check. -* When asking a peer for updates, ZeroNet may see that the other peer has an older version of a file. In this case, ZeroNet sends back the notification of the new version available. The logic in 0.8.0 is generally the same, but some randomization is added which may help in distributing the "update waves" among peers. +* When asking a peer for updates, ZeroNet may detect that the other peer has outdated versions of some files. In this case, ZeroNet sends back the notification of the new versions available. This behaviour was improved in the following ways: + * Prior to 0.8.0 ZeroNet always chose the first 5 files in the list. If peer rejects updating those files for some reason, this strategy led to pushing the same 5 files over and over again on very update check. Now files are selected randomly. This should also improve spreading of "update waves" among peers. + * The engine now keeps track of which files were already sent to a specific peer and doesn't send those again as long as the file timestamp hasn't changed. Now we do not try to resend files if a peer refuses to update them. The watchlist is currently limited to 5000 items. + * The previous two changes should make the send back mechanism more efficient. So number of files to send at once reduced from 5 to 3 to reduce the network load. * ZeroNet now tries harder in delivering updates to more peers in the background. * ZeroNet also makes more efforts of searching the peers before publishing updates. -**Other:** +**Other scalabily and performance issuses:** -* Implemented the log level overriding for separate modules for easier debugging. -* Make the site block check implemented in `ContentFilter` usable from plugins and core modules via `SiteManager.isAddressBlocked()`. -* Fixed possible infinite growing of the `SafeRe` regexp cache by limiting the cache size. +* Fix possible infinite growing of the `SafeRe` regexp cache by limiting the cache size. + +**Changes in API and architecture:** + +* Add method `SiteManager.isAddressBlocked()` available for overriding by content filter plugins in order for other plugins be able explicitly check site block status. +* Hard-coded logic related to the network operation modes (TOR mode, proxies, network address types etc) in many places replaced with appropriate method calls and lots of new methods introduced. These changes are made in order to make it someday possible implementing support for different types of overlay networks (first of all, we mean I2P) in plugins without hardcoding more stuff to the core. A lot of refactoring is yet to come. Changes include but are not limited to: + * `helper.getIpType()` moved to `ConnectionServer.getIpType()`. + * New methods `ConnectionServer.getIpUnreachability()`, `ConnectionServer.isIpReachable()`. + * New methods `Peer.isConnected()`, `Peer.isConnectable()`, `Peer.isReachable()`, `Peer.getIpType()`. + +**Other changes:** + +* Implement the log level overriding for separate modules for easier debugging. ## Docker Image From e000eae04640698d870723e4ad45ef73f643c7dd Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Nov 2021 16:06:06 +0700 Subject: [PATCH 109/114] Add new configuration option: simultaneous_connection_throttle_threshold --- src/Config.py | 2 ++ src/Connection/Connection.py | 14 +++++++++++++- src/Connection/ConnectionServer.py | 17 +++++++++++++++++ src/File/FileServer.py | 13 +++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/Config.py b/src/Config.py index 15a0c87f..a9e05a6f 100644 --- a/src/Config.py +++ b/src/Config.py @@ -285,6 +285,8 @@ class Config(object): self.parser.add_argument('--expose_no_ownership', help='By default, ZeroNet tries checking updates for own sites more frequently. This can be used by a third party for revealing the network addresses of a site owner. If this option is enabled, ZeroNet performs the checks in the same way for any sites.', type='bool', choices=[True, False], default=False) + self.parser.add_argument('--simultaneous_connection_throttle_threshold', help='Throttle opening new connections when the number of outgoing connections in not fully established state exceeds the threshold.', default=15, type=int, metavar='simultaneous_connection_throttle_threshold') + self.parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') self.parser.add_argument('--fileserver_port', help='FileServer bind port (0: randomize)', default=0, type=int, metavar='port') self.parser.add_argument('--fileserver_port_range', help='FileServer randomization range', default="10000-40000", metavar='port') diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index c147ee35..44665b2f 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -17,7 +17,7 @@ from util import helper class Connection(object): __slots__ = ( "sock", "sock_wrapped", "ip", "port", "cert_pin", "target_onion", "id", "protocol", "type", "server", "unpacker", "unpacker_bytes", "req_id", "ip_type", - "handshake", "crypt", "connected", "event_connected", "closed", "start_time", "handshake_time", "last_recv_time", "is_private_ip", "is_tracker_connection", + "handshake", "crypt", "connected", "connecting", "event_connected", "closed", "start_time", "handshake_time", "last_recv_time", "is_private_ip", "is_tracker_connection", "last_message_time", "last_send_time", "last_sent_time", "incomplete_buff_recv", "bytes_recv", "bytes_sent", "cpu_time", "send_lock", "last_ping_delay", "last_req_time", "last_cmd_sent", "last_cmd_recv", "bad_actions", "sites", "name", "waiting_requests", "waiting_streams" ) @@ -50,6 +50,7 @@ class Connection(object): self.crypt = None # Connection encryption method self.sock_wrapped = False # Socket wrapped to encryption + self.connecting = False self.connected = False self.event_connected = gevent.event.AsyncResult() # Solves on handshake received self.closed = False @@ -118,6 +119,15 @@ class Connection(object): # Open connection to peer and wait for handshake def connect(self): + self.connecting = True + try: + return self._connect() + except Exception as err: + self.connecting = False + self.connected = False + raise + + def _connect(self): self.updateOnlineStatus(outgoing_activity=True) if not self.event_connected or self.event_connected.ready(): @@ -236,6 +246,7 @@ class Connection(object): self.protocol = "v2" self.updateName() self.connected = True + self.connecting = False buff_len = 0 req_len = 0 self.unpacker_bytes = 0 @@ -634,6 +645,7 @@ class Connection(object): return False # Already closed self.closed = True self.connected = False + self.connecting = False if self.event_connected: self.event_connected.set(False) diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 9f24e377..9c16b774 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -298,6 +298,7 @@ class ConnectionServer(object): raise Exception("This peer is blacklisted") try: + #self.log.info("Connection to: %s:%s", ip, port) if has_per_site_onion: # Lock connection to site connection = Connection(self, ip, port, target_onion=site_onion, is_tracker_connection=is_tracker_connection) else: @@ -312,6 +313,7 @@ class ConnectionServer(object): raise Exception("Connection event return error") except Exception as err: + #self.log.info("Connection error (%s, %s): %s", ip, port, Debug.formatException(err)) connection.close("%s Connect error: %s" % (ip, Debug.formatException(err))) raise err @@ -429,6 +431,21 @@ class ConnectionServer(object): )) return num_closed + # Returns True if we should slow down opening new connections as at the moment + # there are too many connections being established and not connected completely + # (not entered the message loop yet). + def shouldThrottleNewConnections(self): + threshold = config.simultaneous_connection_throttle_threshold + if len(self.connections) <= threshold: + return False + nr_connections_being_established = 0 + for connection in self.connections[:]: # Make a copy + if connection.connecting and not connection.connected and connection.type == "out": + nr_connections_being_established += 1 + if nr_connections_being_established > threshold: + return True + return False + # Internet outage detection def updateOnlineStatus(self, connection, outgoing_activity=False, successful_activity=False): diff --git a/src/File/FileServer.py b/src/File/FileServer.py index ac4b8c55..66cefd39 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -401,6 +401,9 @@ class FileServer(ConnectionServer): self.sleep(1) self.waitForInternetOnline() + while self.isActiveMode() and self.shouldThrottleNewConnections(): + self.sleep(1) + if not self.isActiveMode(): break @@ -463,6 +466,9 @@ class FileServer(ConnectionServer): self.sleep(short_timeout) self.waitForInternetOnline() + while self.isActiveMode() and self.shouldThrottleNewConnections(): + self.sleep(1) + if not self.isActiveMode(): break @@ -509,6 +515,9 @@ class FileServer(ConnectionServer): while self.isActiveMode(): self.sleep(long_timeout) + while self.isActiveMode() and self.shouldThrottleNewConnections(): + self.sleep(1) + if not self.isActiveMode(): break @@ -591,6 +600,10 @@ class FileServer(ConnectionServer): threshold = self.internet_outage_threshold / 2.0 self.sleep(threshold / 2.0) + + while self.isActiveMode() and self.shouldThrottleNewConnections(): + self.sleep(1) + if not self.isActiveMode(): break From b6b23d0e8e163978b34d22974f1cfc5c9d098393 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Nov 2021 17:15:43 +0700 Subject: [PATCH 110/114] In Site.updater(), properly detect the case when peer has no file at all, not just an older version. --- ZNE-ChangeLog/ChangeLog-0.8.0.md | 3 +- src/Site/Site.py | 47 ++++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/ZNE-ChangeLog/ChangeLog-0.8.0.md b/ZNE-ChangeLog/ChangeLog-0.8.0.md index 568b015d..b51effef 100644 --- a/ZNE-ChangeLog/ChangeLog-0.8.0.md +++ b/ZNE-ChangeLog/ChangeLog-0.8.0.md @@ -23,7 +23,8 @@ * When asking a peer for updates, ZeroNet may detect that the other peer has outdated versions of some files. In this case, ZeroNet sends back the notification of the new versions available. This behaviour was improved in the following ways: * Prior to 0.8.0 ZeroNet always chose the first 5 files in the list. If peer rejects updating those files for some reason, this strategy led to pushing the same 5 files over and over again on very update check. Now files are selected randomly. This should also improve spreading of "update waves" among peers. * The engine now keeps track of which files were already sent to a specific peer and doesn't send those again as long as the file timestamp hasn't changed. Now we do not try to resend files if a peer refuses to update them. The watchlist is currently limited to 5000 items. - * The previous two changes should make the send back mechanism more efficient. So number of files to send at once reduced from 5 to 3 to reduce the network load. + * Prior to 0.8.0 ZeroNet failed to detect the case when remote peer misses a file at all, not just has an older version. Now it is handled properly and those files also can be send back. + * The previous changes should make the send back mechanism more efficient. So number of files to send at once reduced from 5 to 3 to reduce the network load. * ZeroNet now tries harder in delivering updates to more peers in the background. * ZeroNet also makes more efforts of searching the peers before publishing updates. diff --git a/src/Site/Site.py b/src/Site/Site.py index 0cea4733..46e19169 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -6,6 +6,7 @@ import time import random import sys import hashlib +import itertools import collections import base64 @@ -634,32 +635,50 @@ class Site(object): continue # Failed query queried.append(peer) + + modified_contents = [] send_back = [] send_back_limit = config.send_back_limit send_back_skipped = 0 - my_modified = self.content_manager.listModified(since) - num_old_files = 0 - for inner_path, modified in res["modified_files"].items(): # Check if the peer has newer files than we - has_newer = int(modified) > my_modified.get(inner_path, 0) - has_older = int(modified) < my_modified.get(inner_path, 0) - if inner_path not in self.bad_files and not self.content_manager.isArchived(inner_path, modified): - if has_newer: - # We dont have this file or we have older + peer_modified_files = res["modified_files"] + my_modified_files = self.content_manager.listModified(since) + + inner_paths = itertools.chain(peer_modified_files.keys(), my_modified_files.keys()) + seen_inner_paths = {} + for inner_path in inner_paths: # Check if the peer has newer files than we have + if seen_inner_paths.get(inner_path, False): + continue + seen_inner_paths[inner_path] = True + + peer_modified = int(peer_modified_files.get(inner_path, 0)) + my_modified = int(my_modified_files.get(inner_path, 0)) + + diff = peer_modified - my_modified + if diff == 0: + continue + has_newer = diff > 0 + has_older = diff < 0 + + if inner_path not in self.bad_files and not self.content_manager.isArchived(inner_path, peer_modified): + if has_newer: # We don't have this file or we have older version modified_contents.append(inner_path) self.bad_files[inner_path] = self.bad_files.get(inner_path, 1) - if has_older: - if self.checkSendBackLRU(peer, inner_path, modified): + if has_older: # The remote peer doesn't have this file or it has older version + if self.checkSendBackLRU(peer, inner_path, peer_modified): send_back_skipped += 1 else: send_back.append(inner_path) + inner_paths = None + seen_inner_paths = None + if modified_contents: self.log.info("CheckModifications: %s new modified files from %s" % (len(modified_contents), peer)) - modified_contents.sort(key=lambda inner_path: 0 - res["modified_files"][inner_path]) # Download newest first + modified_contents.sort(key=lambda inner_path: 0 - peer_modified_files[inner_path]) # Download newest first for inner_path in modified_contents: self.log.info("CheckModifications: %s: %s > %s" % ( - inner_path, res["modified_files"][inner_path], my_modified.get(inner_path, 0) + inner_path, peer_modified_files.get(inner_path, 0), my_modified_files.get(inner_path, 0) )) t = self.spawn(self.pooledDownloadContent, modified_contents, only_if_bad=True) threads.append(t) @@ -667,12 +686,12 @@ class Site(object): if send_back: self.log.info("CheckModifications: %s has older versions of %s files" % (peer, len(send_back))) if len(send_back) > send_back_limit: - self.log.info("CheckModifications: choosing %s files to publish back" % (send_back_limit)) + self.log.info("CheckModifications: choosing %s random files to publish back" % (send_back_limit)) random.shuffle(send_back) send_back = send_back[0:send_back_limit] for inner_path in send_back: self.log.info("CheckModifications: %s: %s < %s" % ( - inner_path, res["modified_files"][inner_path], my_modified.get(inner_path, 0) + inner_path, peer_modified_files.get(inner_path, 0), my_modified_files.get(inner_path, 0) )) self.spawn(self.publisher, inner_path, [peer], [], 1, save_to_send_back_lru=True) From b7a3aa37e1399a589609b82dd1d8642436638de5 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Fri, 5 Nov 2021 23:38:38 +0700 Subject: [PATCH 111/114] Sidebar Plugin: display more detailed statistics about peers --- plugins/Sidebar/SidebarPlugin.py | 151 ++++++++++++++++++++++++------- src/Connection/Connection.py | 3 +- 2 files changed, 121 insertions(+), 33 deletions(-) diff --git a/plugins/Sidebar/SidebarPlugin.py b/plugins/Sidebar/SidebarPlugin.py index 4ecca75a..f5b40e2d 100644 --- a/plugins/Sidebar/SidebarPlugin.py +++ b/plugins/Sidebar/SidebarPlugin.py @@ -88,31 +88,84 @@ class UiRequestPlugin(object): @PluginManager.registerTo("UiWebsocket") class UiWebsocketPlugin(object): def sidebarRenderPeerStats(self, body, site): - connected = len([peer for peer in list(site.peers.values()) if peer.connection and peer.connection.connected]) - connectable = len([peer_id for peer_id in list(site.peers.keys()) if not peer_id.endswith(":0")]) - onion = len([peer_id for peer_id in list(site.peers.keys()) if ".onion" in peer_id]) - local = len([peer for peer in list(site.peers.values()) if helper.isPrivateIp(peer.ip)]) + # Peers by status peers_total = len(site.peers) + peers_reachable = 0 + peers_connectable = 0 + peers_connected = 0 + peers_failed = 0 + # Peers by type + peers_by_type = {} - # Add myself - if site.isServing(): - peers_total += 1 - if any(site.connection_server.port_opened.values()): - connectable += 1 - if site.connection_server.tor_manager.start_onions: - onion += 1 + type_proper_names = { + 'ipv4': 'IPv4', + 'ipv6': 'IPv6', + 'onion': 'Onion', + 'unknown': 'Unknown' + } + + type_defs = { + 'local-ipv4': { + 'order' : -21, + 'color' : 'yellow' + }, + 'local-ipv6': { + 'order' : -20, + 'color' : 'orange' + }, + 'ipv4': { + 'order' : -11, + 'color' : 'blue' + }, + 'ipv6': { + 'order' : -10, + 'color' : 'darkblue' + }, + 'unknown': { + 'order' : 10, + 'color' : 'red' + }, + } + + for peer in list(site.peers.values()): + # Peers by status + if peer.isConnected(): + peers_connected += 1 + elif peer.isConnectable(): + peers_connectable += 1 + elif peer.isReachable() and not peer.connection_error: + peers_reachable += 1 + elif peer.isReachable() and peer.connection_error: + peers_failed += 1 + # Peers by type + peer_type = peer.getIpType() + peer_readable_type = type_proper_names.get(peer_type, peer_type) + if helper.isPrivateIp(peer.ip): + peer_type = 'local-' + peer.getIpType() + peer_readable_type = 'Local ' + peer_readable_type + peers_by_type[peer_type] = peers_by_type.get(peer_type, { + 'type': peer_type, + 'readable_type': _[peer_readable_type], + 'order': type_defs.get(peer_type, {}).get('order', 0), + 'color': type_defs.get(peer_type, {}).get('color', 'purple'), + 'count': 0 + }) + peers_by_type[peer_type]['count'] += 1 + + ######################################################################## if peers_total: - percent_connected = float(connected) / peers_total - percent_connectable = float(connectable) / peers_total - percent_onion = float(onion) / peers_total + percent_connected = float(peers_connected) / peers_total + percent_connectable = float(peers_connectable) / peers_total + percent_reachable = float(peers_reachable) / peers_total + percent_failed = float(peers_failed) / peers_total + percent_other = min(0.0, 1.0 - percent_connected - percent_connectable - percent_reachable - percent_failed) else: - percent_connectable = percent_connected = percent_onion = 0 - - if local: - local_html = _("
  • {_[Local]}:{local}
    • ") - else: - local_html = "" + percent_connected = 0 + percent_reachable = 0 + percent_connectable = 0 + percent_failed = 0 + percent_other = 0 peer_ips = [peer.key for peer in site.getConnectablePeers(20, allow_private=False)] peer_ips.sort(key=lambda peer_ip: ".onion:" in peer_ip) @@ -127,21 +180,55 @@ class UiWebsocketPlugin(object): {_[Peers]} {_[Copy to clipboard]} -
      -
    • -
    • -
    • +
      • +
      • +
      • +
      • +
        -
      • {_[Connected]}:{connected}
        • -
      • {_[Connectable]}:{connectable}
        • -
      • {_[Onion]}:{onion}
        • - {local_html} -
      • {_[Total]}:{peers_total}
        • +
      • {_[Connected]}:{peers_connected}
        • +
      • {_[Connectable]}:{peers_connectable}
        • +
      • {_[Reachable]}:{peers_reachable}
        • +
      • {_[Failed]}:{peers_failed}
        • +
      • {_[Total peers]}:{peers_total}
      - - """.replace("{local_html}", local_html))) + """)) + + ######################################################################## + + peers_by_type = sorted( + peers_by_type.values(), + key=lambda t: t['order'], + ) + + peers_by_type_graph = '' + peers_by_type_legend = '' + + if peers_total: + for t in peers_by_type: + peers_by_type_graph += """ +
    • + """ % (float(t['count']) * 100.0 / peers_total, t['type'], t["color"], t['readable_type']) + peers_by_type_legend += """ +
    • %s:%s
      • + """ % (t["color"], t['readable_type'], t['count']) + + if peers_by_type_legend != '': + body.append(_(""" +
    • + +
        + %s +
      +
        + %s +
      +
    • + """ % (peers_by_type_graph, peers_by_type_legend))) def sidebarRenderTransferStats(self, body, site): recv = float(site.settings.get("bytes_recv", 0)) / 1024 / 1024 @@ -682,7 +769,7 @@ class UiWebsocketPlugin(object): # Create position array lat, lon = loc["lat"], loc["lon"] latlon = "%s,%s" % (lat, lon) - if latlon in placed and helper.getIpType(peer.ip) == "ipv4": # Dont place more than 1 bar to same place, fake repos using ip address last two part + if latlon in placed and peer.getIpType() == "ipv4": # Dont place more than 1 bar to same place, fake repos using ip address last two part lat += float(128 - int(peer.ip.split(".")[-2])) / 50 lon += float(128 - int(peer.ip.split(".")[-1])) / 50 latlon = "%s,%s" % (lat, lon) diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 44665b2f..ad1312f2 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -621,7 +621,8 @@ class Connection(object): self.waiting_requests[self.req_id] = {"evt": event, "cmd": cmd} if stream_to: self.waiting_streams[self.req_id] = stream_to - self.send(data) # Send request + if not self.send(data): # Send request + return False res = event.get() # Wait until event solves return res From f7372fc393021eddcc81dec22b1c604e73f2b34d Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 6 Nov 2021 13:16:43 +0700 Subject: [PATCH 112/114] Remove __slots__ from Peer, the slot system doesn't work well with classes extended by plugins anyway --- src/Peer/Peer.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 6e4863e0..43c2932f 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -20,12 +20,6 @@ if config.use_tempfiles: # Communicate remote peers @PluginManager.acceptPlugins class Peer(object): - __slots__ = ( - "ip", "port", "site", "key", "connection", "connection_server", "time_found", "time_response", "time_hashfield", - "time_added", "has_hashfield", "is_tracker_connection", "time_my_hashfield_sent", "last_ping", "reputation", - "last_content_json_update", "hashfield", "connection_error", "hash_failed", "download_bytes", "download_time" - ) - def __init__(self, ip, port, site=None, connection_server=None): self.ip = ip self.port = port From 348a4b08650a255a64359b07e525e27aaba8e077 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 6 Nov 2021 20:33:19 +0700 Subject: [PATCH 113/114] Add support for building ubuntu:20.04-based docker images + some automation --- Dockerfile | 43 +--------------- build-docker-images.sh | 11 +++++ dockerfiles/Dockerfile.alpine | 1 + dockerfiles/Dockerfile.alpine3.13 | 44 +++++++++++++++++ dockerfiles/Dockerfile.ubuntu | 1 + dockerfiles/Dockerfile.ubuntu20.04 | 44 +++++++++++++++++ dockerfiles/gen-dockerfiles.sh | 34 +++++++++++++ .../zeronet-Dockerfile | 28 ++++++----- install-dep-packages.sh | 49 +++++++++++++++++++ 9 files changed, 200 insertions(+), 55 deletions(-) mode change 100644 => 120000 Dockerfile create mode 100755 build-docker-images.sh create mode 120000 dockerfiles/Dockerfile.alpine create mode 100644 dockerfiles/Dockerfile.alpine3.13 create mode 120000 dockerfiles/Dockerfile.ubuntu create mode 100644 dockerfiles/Dockerfile.ubuntu20.04 create mode 100755 dockerfiles/gen-dockerfiles.sh rename Dockerfile.arm64v8 => dockerfiles/zeronet-Dockerfile (55%) create mode 100755 install-dep-packages.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index bc834293..00000000 --- a/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -FROM alpine:3.13 - -# Base settings -ENV HOME /root - -# Install packages - -RUN apk --update --no-cache --no-progress add \ - python3 python3-dev py3-pip \ - libffi-dev musl-dev \ - gcc g++ make \ - automake autoconf libtool \ - openssl \ - tor - -COPY requirements.txt /root/requirements.txt - -RUN pip3 install -r /root/requirements.txt \ - && apk del python3-dev gcc libffi-dev musl-dev make \ - && echo "ControlPort 9051" >> /etc/tor/torrc \ - && echo "CookieAuthentication 1" >> /etc/tor/torrc - -RUN python3 -V \ - && python3 -m pip list \ - && tor --version \ - && openssl version - -# Add Zeronet source - -COPY . /root -VOLUME /root/data - -# Control if Tor proxy is started -ENV ENABLE_TOR false - -WORKDIR /root - -# Set upstart command -CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 - -# Expose ports -EXPOSE 43110 26552 diff --git a/Dockerfile b/Dockerfile new file mode 120000 index 00000000..ae9f3313 --- /dev/null +++ b/Dockerfile @@ -0,0 +1 @@ +dockerfiles/Dockerfile.alpine \ No newline at end of file diff --git a/build-docker-images.sh b/build-docker-images.sh new file mode 100755 index 00000000..88dec2fb --- /dev/null +++ b/build-docker-images.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -e + +prefix="${1:-local/}" + +for dokerfile in dockerfiles/Dockerfile.* ; do + suffix="`echo "$dokerfile" | sed 's/.*\/Dockerfile\.//'`" + image_name="${prefix}zeronet:$suffix" + echo "DOCKER BUILD $image_name" + docker build -f "$dokerfile" -t "$image_name" . +done diff --git a/dockerfiles/Dockerfile.alpine b/dockerfiles/Dockerfile.alpine new file mode 120000 index 00000000..0f848cc8 --- /dev/null +++ b/dockerfiles/Dockerfile.alpine @@ -0,0 +1 @@ +Dockerfile.alpine3.13 \ No newline at end of file diff --git a/dockerfiles/Dockerfile.alpine3.13 b/dockerfiles/Dockerfile.alpine3.13 new file mode 100644 index 00000000..79f15b9b --- /dev/null +++ b/dockerfiles/Dockerfile.alpine3.13 @@ -0,0 +1,44 @@ +# THIS FILE IS AUTOGENERATED BY gen-dockerfiles.sh. +# SEE zeronet-Dockerfile FOR THE SOURCE FILE. + +FROM alpine:3.13 + +# Base settings +ENV HOME /root + +# Install packages + +# Install packages + +COPY install-dep-packages.sh /root/install-dep-packages.sh + +RUN /root/install-dep-packages.sh install + +COPY requirements.txt /root/requirements.txt + +RUN pip3 install -r /root/requirements.txt \ + && /root/install-dep-packages.sh remove-makedeps \ + && echo "ControlPort 9051" >> /etc/tor/torrc \ + && echo "CookieAuthentication 1" >> /etc/tor/torrc + +RUN python3 -V \ + && python3 -m pip list \ + && tor --version \ + && openssl version + +# Add Zeronet source + +COPY . /root +VOLUME /root/data + +# Control if Tor proxy is started +ENV ENABLE_TOR false + +WORKDIR /root + +# Set upstart command +CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 + +# Expose ports +EXPOSE 43110 26552 + diff --git a/dockerfiles/Dockerfile.ubuntu b/dockerfiles/Dockerfile.ubuntu new file mode 120000 index 00000000..29adf7ef --- /dev/null +++ b/dockerfiles/Dockerfile.ubuntu @@ -0,0 +1 @@ +Dockerfile.ubuntu20.04 \ No newline at end of file diff --git a/dockerfiles/Dockerfile.ubuntu20.04 b/dockerfiles/Dockerfile.ubuntu20.04 new file mode 100644 index 00000000..bc32cf86 --- /dev/null +++ b/dockerfiles/Dockerfile.ubuntu20.04 @@ -0,0 +1,44 @@ +# THIS FILE IS AUTOGENERATED BY gen-dockerfiles.sh. +# SEE zeronet-Dockerfile FOR THE SOURCE FILE. + +FROM ubuntu:20.04 + +# Base settings +ENV HOME /root + +# Install packages + +# Install packages + +COPY install-dep-packages.sh /root/install-dep-packages.sh + +RUN /root/install-dep-packages.sh install + +COPY requirements.txt /root/requirements.txt + +RUN pip3 install -r /root/requirements.txt \ + && /root/install-dep-packages.sh remove-makedeps \ + && echo "ControlPort 9051" >> /etc/tor/torrc \ + && echo "CookieAuthentication 1" >> /etc/tor/torrc + +RUN python3 -V \ + && python3 -m pip list \ + && tor --version \ + && openssl version + +# Add Zeronet source + +COPY . /root +VOLUME /root/data + +# Control if Tor proxy is started +ENV ENABLE_TOR false + +WORKDIR /root + +# Set upstart command +CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 + +# Expose ports +EXPOSE 43110 26552 + diff --git a/dockerfiles/gen-dockerfiles.sh b/dockerfiles/gen-dockerfiles.sh new file mode 100755 index 00000000..75a6edf6 --- /dev/null +++ b/dockerfiles/gen-dockerfiles.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +set -e + +die() { + echo "$@" > /dev/stderr + exit 1 +} + +for os in alpine:3.13 ubuntu:20.04 ; do + prefix="`echo "$os" | sed -e 's/://'`" + short_prefix="`echo "$os" | sed -e 's/:.*//'`" + + zeronet="zeronet-Dockerfile" + + dockerfile="Dockerfile.$prefix" + dockerfile_short="Dockerfile.$short_prefix" + + echo "GEN $dockerfile" + + if ! test -f "$zeronet" ; then + die "No such file: $zeronet" + fi + + echo "\ +# THIS FILE IS AUTOGENERATED BY gen-dockerfiles.sh. +# SEE $zeronet FOR THE SOURCE FILE. + +FROM $os + +`cat "$zeronet"` +" > "$dockerfile.tmp" && mv "$dockerfile.tmp" "$dockerfile" && ln -s -f "$dockerfile" "$dockerfile_short" +done + diff --git a/Dockerfile.arm64v8 b/dockerfiles/zeronet-Dockerfile similarity index 55% rename from Dockerfile.arm64v8 rename to dockerfiles/zeronet-Dockerfile index d27b7620..92c67c84 100644 --- a/Dockerfile.arm64v8 +++ b/dockerfiles/zeronet-Dockerfile @@ -1,34 +1,36 @@ -FROM alpine:3.12 - -#Base settings +# Base settings ENV HOME /root +# Install packages + +COPY install-dep-packages.sh /root/install-dep-packages.sh + +RUN /root/install-dep-packages.sh install + COPY requirements.txt /root/requirements.txt -#Install ZeroNet -RUN apk --update --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \ - && pip3 install -r /root/requirements.txt \ - && apk del python3-dev gcc libffi-dev musl-dev make \ +RUN pip3 install -r /root/requirements.txt \ + && /root/install-dep-packages.sh remove-makedeps \ && echo "ControlPort 9051" >> /etc/tor/torrc \ && echo "CookieAuthentication 1" >> /etc/tor/torrc - + RUN python3 -V \ && python3 -m pip list \ && tor --version \ && openssl version -#Add Zeronet source +# Add Zeronet source + COPY . /root VOLUME /root/data -#Control if Tor proxy is started +# Control if Tor proxy is started ENV ENABLE_TOR false WORKDIR /root -#Set upstart command +# Set upstart command CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 -#Expose ports +# Expose ports EXPOSE 43110 26552 - diff --git a/install-dep-packages.sh b/install-dep-packages.sh new file mode 100755 index 00000000..655a33aa --- /dev/null +++ b/install-dep-packages.sh @@ -0,0 +1,49 @@ +#!/bin/sh +set -e + +do_alpine() { + local deps="python3 py3-pip openssl tor" + local makedeps="python3-dev gcc g++ libffi-dev musl-dev make automake autoconf libtool" + + case "$1" in + install) + apk --update --no-cache --no-progress add $deps $makedeps + ;; + remove-makedeps) + apk del $makedeps + ;; + esac +} + +do_ubuntu() { + local deps="python3 python3-pip openssl tor" + local makedeps="python3-dev gcc g++ libffi-dev make automake autoconf libtool" + + case "$1" in + install) + apt-get update && \ + apt-get install --no-install-recommends -y $deps $makedeps && \ + rm -rf /var/lib/apt/lists/* + ;; + remove-makedeps) + apt-get remove -y $makedeps + ;; + esac +} + +if test -f /etc/os-release ; then + . /etc/os-release +elif test -f /usr/lib/os-release ; then + . /usr/lib/os-release +else + echo "No such file: /etc/os-release" > /dev/stderr + exit 1 +fi + +case "$ID" in + ubuntu) do_ubuntu "$@" ;; + alpine) do_alpine "$@" ;; + *) + echo "Unsupported OS ID: $ID" > /dev/stderr + exit 1 +esac From 5c8bbe5801ea499e59d89274686de2662a6d2560 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Sat, 6 Nov 2021 22:00:46 +0700 Subject: [PATCH 114/114] build-docker-images.sh: push to Docker Hub as well --- build-docker-images.sh | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/build-docker-images.sh b/build-docker-images.sh index 88dec2fb..8eff34f4 100755 --- a/build-docker-images.sh +++ b/build-docker-images.sh @@ -1,11 +1,32 @@ #!/bin/sh set -e +arg_push= + +case "$1" in + --push) arg_push=y ; shift ;; +esac + +default_suffix=alpine prefix="${1:-local/}" for dokerfile in dockerfiles/Dockerfile.* ; do suffix="`echo "$dokerfile" | sed 's/.*\/Dockerfile\.//'`" image_name="${prefix}zeronet:$suffix" + + latest="" + t_latest="" + if [ "$suffix" = "$default_suffix" ] ; then + latest="${prefix}zeronet:latest" + t_latest="-t ${latest}" + fi + echo "DOCKER BUILD $image_name" - docker build -f "$dokerfile" -t "$image_name" . + docker build -f "$dokerfile" -t "$image_name" $t_latest . + if [ -n "$arg_push" ] ; then + docker push "$image_name" + if [ -n "$latest" ] ; then + docker push "$latest" + fi + fi done