From 27f2c445324039c6d3632c01ffba131ce98e7f43 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sun, 26 Aug 2018 02:45:37 +0200 Subject: [PATCH] AnnounceShare plugin --- plugins/AnnounceShare/AnnounceSharePlugin.py | 153 ++++++++++++++++++ .../AnnounceShare/Test/TestAnnounceShare.py | 29 ++++ plugins/AnnounceShare/Test/conftest.py | 3 + plugins/AnnounceShare/Test/pytest.ini | 5 + plugins/AnnounceShare/__init__.py | 1 + 5 files changed, 191 insertions(+) create mode 100644 plugins/AnnounceShare/AnnounceSharePlugin.py create mode 100644 plugins/AnnounceShare/Test/TestAnnounceShare.py create mode 100644 plugins/AnnounceShare/Test/conftest.py create mode 100644 plugins/AnnounceShare/Test/pytest.ini create mode 100644 plugins/AnnounceShare/__init__.py diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py new file mode 100644 index 00000000..ffbed91b --- /dev/null +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -0,0 +1,153 @@ +import time +import os +import logging +import json +import atexit + +import gevent + +from Config import config +from Plugin import PluginManager +from util import helper +import util + + +class TrackerStorage(object): + def __init__(self): + self.log = logging.getLogger("TrackerStorage") + self.file_path = "%s/trackers.json" % config.data_dir + self.file_content = self.load() + + self.time_discover = 0.0 + atexit.register(self.save) + + def getDefaultFile(self): + return {"shared": {}} + + def onTrackerFound(self, tracker_address, type="shared", my=False): + trackers = self.getTrackers() + if tracker_address not in trackers: + trackers[tracker_address] = { + "time_added": time.time(), + "time_success": 0, + "latency": 99.0, + "num_error": 0, + "my": False + } + trackers[tracker_address]["time_found"] = time.time() + trackers[tracker_address]["my"] = my + self.log.debug("New tracker found: %s" % tracker_address) + + def onTrackerSuccess(self, tracker_address, latency): + trackers = self.getTrackers() + if tracker_address not in trackers: + return False + + trackers[tracker_address]["latency"] = latency + trackers[tracker_address]["time_success"] = time.time() + trackers[tracker_address]["num_error"] = 0 + + def onTrackerError(self, tracker_address): + trackers = self.getTrackers() + if tracker_address not in trackers: + return False + + trackers[tracker_address]["time_error"] = time.time() + trackers[tracker_address]["num_error"] += 1 + + if trackers[tracker_address]["num_error"] > 30 and trackers[tracker_address]["time_success"] < time.time() - 60 * 60 * 24: + self.log.debug("Tracker %s looks down, removing." % tracker_address) + del trackers[tracker_address] + + def getTrackers(self, type="shared"): + return self.file_content.setdefault(type, {}) + + def getWorkingTrackers(self, type): + trackers = { + key: tracker for key, tracker in self.getTrackers(type).iteritems() + if tracker["time_success"] > time.time() - 60 * 60 * 24 + } + return trackers + + def load(self): + if not os.path.isfile(self.file_path): + open(self.file_path, "w").write("{}") + return self.getDefaultFile() + try: + return json.load(open(self.file_path)) + except Exception as err: + self.log.error("Error loading trackers list: %s" % err) + return self.getDefaultFile() + + def save(self): + s = time.time() + helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True)) + self.log.debug("Saved in %.3fs" % (time.time() - s)) + + def discoverTrackers(self, peers): + s = time.time() + num_success = 0 + for peer in peers: + if peer.connection and peer.connection.handshake.get("rev", 0) < 3560: + continue # Not supported + + res = peer.request("getTrackers") + if not res or "error" in res: + continue + + num_success += 1 + for tracker_address in res["trackers"]: + self.onTrackerFound(tracker_address) + + if not num_success and len(peers) < 20: + self.time_discover = 0.0 + + if num_success: + self.save() + + if config.verbose: + self.log.debug("Trackers discovered from %s/%s peers in %.3fs" % (num_success, len(peers), time.time() - s)) + + +tracker_storage = TrackerStorage() + + +@PluginManager.registerTo("SiteAnnouncer") +class SiteAnnouncerPlugin(object): + def getTrackers(self): + if tracker_storage.time_discover < time.time() - 5 * 60: + tracker_storage.time_discover = time.time() + gevent.spawn(tracker_storage.discoverTrackers, self.site.getConnectedPeers()) + trackers = super(SiteAnnouncerPlugin, self).getTrackers() + shared_trackers = tracker_storage.getTrackers("shared").keys() + if shared_trackers: + return trackers + shared_trackers + else: + return trackers + + def announceTracker(self, tracker, *args, **kwargs): + res = super(SiteAnnouncerPlugin, self).announceTracker(tracker, *args, **kwargs) + if res: + latency = res + tracker_storage.onTrackerSuccess(tracker, latency) + else: + tracker_storage.onTrackerError(tracker) + + return res + + +@PluginManager.registerTo("FileRequest") +class FileRequestPlugin(object): + def actionGetTrackers(self, params): + shared_trackers = tracker_storage.getWorkingTrackers("shared").keys() + self.response({"trackers": shared_trackers}) + + +@PluginManager.registerTo("FileServer") +class FileServerPlugin(object): + def openport(self, *args, **kwargs): + res = super(FileServerPlugin, self).openport(*args, **kwargs) + if res and not config.tor == "always" and "Bootstrapper" in PluginManager.plugin_manager.plugin_names: + my_tracker_address = "zero://%s:%s" % (config.ip_external, config.fileserver_port) + tracker_storage.onTrackerFound(my_tracker_address, my=True) + return res diff --git a/plugins/AnnounceShare/Test/TestAnnounceShare.py b/plugins/AnnounceShare/Test/TestAnnounceShare.py new file mode 100644 index 00000000..060e3cdb --- /dev/null +++ b/plugins/AnnounceShare/Test/TestAnnounceShare.py @@ -0,0 +1,29 @@ +import time +import copy + +import gevent +import pytest +import mock + +from AnnounceShare import AnnounceSharePlugin +from File import FileServer +from Peer import Peer +from Test import Spy + + +@pytest.mark.usefixtures("resetSettings") +@pytest.mark.usefixtures("resetTempSettings") +class TestAnnounceShare: + def testAnnounceList(self, file_server): + peer = Peer("127.0.0.1", 1544, connection_server=file_server) + assert peer.request("getTrackers")["trackers"] == [] + + tracker_storage = AnnounceSharePlugin.tracker_storage + tracker_storage.onTrackerFound("zero://127.0.0.1:15441") + assert peer.request("getTrackers")["trackers"] == [] + + # It needs to have at least one successfull announce to be shared to other peers + tracker_storage.onTrackerSuccess("zero://127.0.0.1:15441", 1.0) + assert peer.request("getTrackers")["trackers"] == ["zero://127.0.0.1:15441"] + + diff --git a/plugins/AnnounceShare/Test/conftest.py b/plugins/AnnounceShare/Test/conftest.py new file mode 100644 index 00000000..5abd4dd6 --- /dev/null +++ b/plugins/AnnounceShare/Test/conftest.py @@ -0,0 +1,3 @@ +from src.Test.conftest import * + +from Config import config diff --git a/plugins/AnnounceShare/Test/pytest.ini b/plugins/AnnounceShare/Test/pytest.ini new file mode 100644 index 00000000..d09210d1 --- /dev/null +++ b/plugins/AnnounceShare/Test/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +python_files = Test*.py +addopts = -rsxX -v --durations=6 +markers = + webtest: mark a test as a webtest. \ No newline at end of file diff --git a/plugins/AnnounceShare/__init__.py b/plugins/AnnounceShare/__init__.py new file mode 100644 index 00000000..f55cb2c6 --- /dev/null +++ b/plugins/AnnounceShare/__init__.py @@ -0,0 +1 @@ +import AnnounceSharePlugin