From 0de6496f96316514a07c4b1cbaf4a9f29a2411cb Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Thu, 10 Sep 2015 23:25:09 +0200 Subject: [PATCH] Rev390, Fix sidebar error on non locatable IPs, Configurable bootstrap torrent trackers, Multi-line config file settings, Evenly distributed tracker announce to work better on passive connections, Avoid iframe sandbox escape by using nonces, Better html error messages, Display proper error on invalid startup parameters --- plugins/Sidebar/SidebarPlugin.py | 2 +- src/Config.py | 16 ++++++++-- src/File/FileServer.py | 22 +++++++++++-- src/Site/Site.py | 44 ++++++++++++++++---------- src/Site/SiteManager.py | 20 ------------ src/Ui/UiRequest.py | 54 +++++++++++++++++++++++++------- src/Ui/UiServer.py | 1 + src/Ui/media/all.css | 6 ++-- src/Ui/template/wrapper.html | 6 +++- src/Worker/WorkerManager.py | 2 +- src/main.py | 2 ++ 11 files changed, 117 insertions(+), 58 deletions(-) diff --git a/plugins/Sidebar/SidebarPlugin.py b/plugins/Sidebar/SidebarPlugin.py index d3490cd4..1724a078 100644 --- a/plugins/Sidebar/SidebarPlugin.py +++ b/plugins/Sidebar/SidebarPlugin.py @@ -376,7 +376,7 @@ class UiWebsocketPlugin(object): else: loc = geodb.get(peer.ip) loc_cache[peer.ip] = loc - if not loc: + if not loc or "location" not in loc: continue # Create position array diff --git a/src/Config.py b/src/Config.py index fe0079e9..e850a8a8 100644 --- a/src/Config.py +++ b/src/Config.py @@ -8,7 +8,7 @@ class Config(object): def __init__(self, argv): self.version = "0.3.2" - self.rev = 378 + self.rev = 390 self.argv = argv self.action = None self.createParser() @@ -29,6 +29,14 @@ class Config(object): # Create command line arguments def createArguments(self): + trackers = [ + "udp://open.demonii.com:1337", + "udp://tracker.leechers-paradise.org:6969", + "udp://9.rarbg.com:2710", + "http://tracker.aletorrenty.pl:2710/announce", + "http://retracker.telecom.kz/announce", + "http://torrent.gresille.org/announce" + ] # Platform specific if sys.platform.startswith("win"): coffeescript = "type %s | tools\\coffee\\coffee.cmd" @@ -122,7 +130,8 @@ class Config(object): self.parser.add_argument('--fileserver_port', help='FileServer bind port', default=15441, type=int, metavar='port') self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true') self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port') - self.parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip') + self.parser.add_argument('--ip_external', help='Set reported external ip (tested on start if None)', metavar='ip') + self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*') self.parser.add_argument('--use_openssl', help='Use OpenSSL liblary for speedup', type='bool', choices=[True, False], default=use_openssl) self.parser.add_argument('--disable_encryption', help='Disable connection encryption', action='store_true') @@ -238,7 +247,8 @@ class Config(object): if section != "global": # If not global prefix key with section key = section + "_" + key if val: - argv.insert(1, val) + for line in val.strip().split("\n"): # Allow multi-line values + argv.insert(1, line) argv.insert(1, "--%s" % key) return argv diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 30c6e9da..0f59a43d 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -170,11 +170,15 @@ class FileServer(ConnectionServer): # Announce sites every 20 min def announceSites(self): import gc + first_announce = True # First start while 1: - time.sleep(20 * 60) # Announce sites every 20 min + # Sites healthcare for address, site in self.sites.items(): if site.settings["serving"]: - site.announce() # Announce site to tracker + if first_announce: # Announce to all trackers on startup + site.announce() + else: # If not first run only use PEX + site.announcePex() # Reset bad file retry counter for inner_path in site.bad_files: @@ -195,6 +199,20 @@ class FileServer(ConnectionServer): site = None gc.collect() # Implicit grabage collection + # Find new peers + for tracker_i in range(len(config.trackers)): + time.sleep(60 * 20 / len(config.trackers)) # Query all trackers one-by-one in 20 minutes evenly distributed + + for address, site in self.sites.items(): + site.announce(num = 1, pex = False) + time.sleep(2) + + first_announce = False + + + + + # Detects if computer back from wakeup def wakeupWatcher(self): last_time = time.time() diff --git a/src/Site/Site.py b/src/Site/Site.py index cdee33a0..6152e0d8 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -38,6 +38,7 @@ class Site: self.peers = {} # Key: ip:port, Value: Peer.Peer self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself) self.last_announce = 0 # Last announce time to tracker + self.last_tracker_id = random.randint(0, 10) # Last announced tracker id self.worker_manager = WorkerManager(self) # Handle site download from other peers self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept) self.content_updated = None # Content.js update time @@ -510,12 +511,13 @@ class Site: # Gather peers from tracker # Return: Complete time or False on error - def announceTracker(self, protocol, ip, port, fileserver_port, address_hash, my_peer_id): + def announceTracker(self, protocol, address, fileserver_port, address_hash, my_peer_id): s = time.time() if protocol == "udp": # Udp tracker if config.disable_udp: return False # No udp supported - tracker = UdpTrackerClient(ip, port) + ip, port = address.split(":") + tracker = UdpTrackerClient(ip, int(port)) tracker.peer_port = fileserver_port try: tracker.connect() @@ -535,7 +537,7 @@ class Site: } req = None try: - url = "http://" + ip + "?" + urllib.urlencode(params) + url = "http://" + address + "?" + urllib.urlencode(params) # Load url with gevent.Timeout(10, False): # Make sure of timeout req = urllib2.urlopen(url, timeout=8) @@ -577,10 +579,17 @@ class Site: return time.time() - s # Add myself and get other peers from tracker - def announce(self, force=False): + def announce(self, force=False, num=5, pex=True): if time.time() < self.last_announce + 30 and not force: return # No reannouncing within 30 secs self.last_announce = time.time() + + trackers = config.trackers + if num == 1: # Only announce on one tracker, increment the queried tracker id + self.last_tracker_id += 1 + self.last_tracker_id = self.last_tracker_id % len(trackers) + trackers = [trackers[self.last_tracker_id]] # We only going to use this one + errors = [] slow = [] address_hash = hashlib.sha1(self.address).hexdigest() # Site address hash @@ -595,39 +604,42 @@ class Site: announced = 0 threads = [] - for protocol, ip, port in SiteManager.TRACKERS: # Start announce threads - thread = gevent.spawn(self.announceTracker, protocol, ip, port, fileserver_port, address_hash, my_peer_id) + for tracker in trackers: # Start announce threads + protocol, address = tracker.split("://") + thread = gevent.spawn(self.announceTracker, protocol, address, fileserver_port, address_hash, my_peer_id) threads.append(thread) - thread.ip = ip + thread.address = address thread.protocol = protocol + if len(threads) > num: break # Announce limit gevent.joinall(threads) # Wait for announce finish for thread in threads: if thread.value: if thread.value > 1: - slow.append("%.2fs %s://%s" % (thread.value, thread.protocol, thread.ip)) + slow.append("%.2fs %s://%s" % (thread.value, thread.protocol, thread.address)) announced += 1 else: - errors.append("%s://%s" % (thread.protocol, thread.ip)) + errors.append("%s://%s" % (thread.protocol, thread.address)) # Save peers num self.settings["peers"] = len(self.peers) self.saveSettings() - if len(errors) < len(SiteManager.TRACKERS): # Less errors than total tracker nums + if len(errors) < min(num, len(trackers)): # Less errors than total tracker nums self.log.debug( "Announced port %s to %s trackers in %.3fs, errors: %s, slow: %s" % (fileserver_port, announced, time.time() - s, errors, slow) ) else: - self.log.error("Announced to %s trackers in %.3fs, failed" % (announced, time.time() - s)) + self.log.error("Announce to %s trackers in %.3fs, failed" % (announced, time.time() - s)) - if not [peer for peer in self.peers.values() if peer.connection and peer.connection.connected]: - # If no connected peer yet then wait for connections - gevent.spawn_later(3, self.announcePex, need_num=10) # Spawn 3 secs later - else: # Else announce immediately - self.announcePex() + if pex: + if not [peer for peer in self.peers.values() if peer.connection and peer.connection.connected]: + # If no connected peer yet then wait for connections + gevent.spawn_later(3, self.announcePex, need_num=10) # Spawn 3 secs later + else: # Else announce immediately + self.announcePex() # Keep connections to get the updates (required for passive clients) def needConnections(self, num=3): diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 1b61a5e3..18f9d278 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -6,26 +6,6 @@ import os from Plugin import PluginManager from Config import config -TRACKERS = [ - ("udp", "open.demonii.com", 1337), - # ("udp", "sugoi.pomf.se", 2710), - # ("udp", "tracker.coppersurfer.tk", 80), - ("udp", "tracker.leechers-paradise.org", 6969), - ("udp", "9.rarbg.com", 2710), - # ("udp", "www.eddie4.nl", 6969), - # ("udp", "trackr.sytes.net", 80), - # ("udp", "tracker4.piratux.com", 6969) - # ("http", "exodus.desync.com:80/announce", None), Off - ("http", "tracker.aletorrenty.pl:2710/announce", None), - # ("http", "torrent.gresille.org/announce", None), # Slow - # ("http", "announce.torrentsmd.com:6969/announce", None), # Off - # ("http", "i.bandito.org/announce", None), # Off - ("http", "retracker.telecom.kz/announce", None), - ("http", "torrent.gresille.org/announce", None), - -] - - @PluginManager.acceptPlugins class SiteManager(object): diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 52f6a42a..a7aa844d 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -4,6 +4,8 @@ import os import mimetypes import json import cgi +import string +import random from Config import config from Site import SiteManager @@ -69,7 +71,7 @@ class UiRequest(object): return self.actionConsole() # Site media wrapper else: - if self.get.get("wrapper") == "False": + if self.get.get("wrapper_nonce"): return self.actionSiteMedia("/media" + path) # Only serve html files with frame else: body = self.actionWrapper(path) @@ -202,7 +204,6 @@ class UiRequest(object): else: # Bad url return False - def renderWrapper(self, site, path, inner_path, title, extra_headers): file_inner_path = inner_path if not file_inner_path: @@ -219,10 +220,12 @@ class UiRequest(object): body_style = "" meta_tags = "" + wrapper_nonce = self.getWrapperNonce() + if self.env.get("QUERY_STRING"): - query_string = "?" + self.env["QUERY_STRING"] + "&wrapper=False" + query_string = "?%s&wrapper_nonce=%s" % (self.env["QUERY_STRING"], wrapper_nonce) else: - query_string = "?wrapper=False" + query_string = "?wrapper_nonce=%s" % wrapper_nonce if self.isProxyRequest(): # Its a remote proxy request if self.env["REMOTE_ADDR"] == "127.0.0.1": # Local client, the server address also should be 127.0.0.1 @@ -260,6 +263,13 @@ class UiRequest(object): homepage=homepage ) + # Create a new wrapper nonce that allows to get one html file without the wrapper + def getWrapperNonce(self): + wrapper_nonce = ''.join( + random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24) + ) + self.server.wrapper_nonces.append(wrapper_nonce) + return wrapper_nonce # Returns if media request allowed from that referer def isMediaRequestAllowed(self, site_address, referer): @@ -276,6 +286,14 @@ class UiRequest(object): match = re.match("/media/(?P
[A-Za-z0-9\._-]+)/(?P.*)", path) + # Check wrapper nonce + content_type = self.getContentType(path) + if "htm" in content_type: # Valid nonce must present to render html files + wrapper_nonce = self.get["wrapper_nonce"] + if wrapper_nonce not in self.server.wrapper_nonces: + return self.error403("Wrapper nonce error.") + self.server.wrapper_nonces.remove(self.get["wrapper_nonce"]) + referer = self.env.get("HTTP_REFERER") if referer and match: # Only allow same site to receive media if not self.isMediaRequestAllowed(match.group("address"), referer): @@ -421,24 +439,38 @@ class UiRequest(object): # - Errors - # Send bad request error - def error400(self): + def error400(self, message=""): self.sendHeader(400) - return "Bad Request" + return self.formatError("Bad Request", message) # You are not allowed to access this - def error403(self, message="Forbidden"): + def error403(self, message=""): self.sendHeader(403) - return message + return self.formatError("Forbidden", message) # Send file not found error - def error404(self, path=None): + def error404(self, path=""): self.sendHeader(404) - return "Not Found: %s" % path.encode("utf8") + return self.formatError("Not Found", path.encode("utf8")) # Internal server error def error500(self, message=":("): self.sendHeader(500) - return "

Server error

%s" % cgi.escape(message) + return self.formatError("Server error", cgi.escape(message)) + + def formatError(self, title, message): + details = {key: val for key, val in self.env.items() if hasattr(val, "endswith") and "COOKIE" not in key } + return """ +

%s

+

%s

+

Please report it if you think this an error.

+

Details:

+ 
%s
+ + """ % (title, message, json.dumps(details, indent=4)) # - Reload for eaiser developing - diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index 280304d6..797a6355 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -55,6 +55,7 @@ class UiServer: self.port = config.ui_port if self.ip == "*": self.ip = "" # Bind all + self.wrapper_nonces = [] self.sites = SiteManager.site_manager.list() self.log = logging.getLogger(__name__) diff --git a/src/Ui/media/all.css b/src/Ui/media/all.css index 29b470d8..a649c605 100644 --- a/src/Ui/media/all.css +++ b/src/Ui/media/all.css @@ -35,7 +35,7 @@ a { color: black } display: block; width: 80px; height: 80px; -webkit-transition: background-color 0.2s, box-shadow 0.5s; -moz-transition: background-color 0.2s, box-shadow 0.5s; -o-transition: background-color 0.2s, box-shadow 0.5s; -ms-transition: background-color 0.2s, box-shadow 0.5s; transition: background-color 0.2s, box-shadow 0.5s ; -webkit-transform: scale(0.6); -moz-transform: scale(0.6); -o-transform: scale(0.6); -ms-transform: scale(0.6); transform: scale(0.6) ; margin-left: -20px; margin-top: -20px; /* 2x size to prevent blur on anim */ /*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* -webkit-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -moz-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -o-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -ms-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1) ; */ } -.fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px } +.fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; -o-backface-visibility: hidden; -ms-backface-visibility: hidden; backface-visibility: hidden ; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px } .fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px } .fixbutton-bg:hover { background-color: #AF3BFF } .fixbutton-bg:active { background-color: #9E2FEA; top: 1px; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none } @@ -45,7 +45,7 @@ a { color: black } .notifications { position: absolute; top: 0px; right: 80px; display: inline-block; z-index: 999; white-space: nowrap } .notification { - position: relative; float: right; clear: both; margin: 10px; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; -o-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box ; overflow: hidden; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; padding-bottom: 5px; + position: relative; float: right; clear: both; margin: 10px; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; -o-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box ; overflow: hidden; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; -o-backface-visibility: hidden; -ms-backface-visibility: hidden; backface-visibility: hidden ; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; padding-bottom: 5px; color: #4F4F4F; font-family: 'Lucida Grande', 'Segoe UI', Helvetica, Arial, sans-serif; font-size: 14px; line-height: 20px; /*border: 1px solid rgba(210, 206, 205, 0.2)*/ } .notification-icon { @@ -112,7 +112,7 @@ a { color: black } .flipper-container { width: 40px; height: 40px; position: absolute; top: 0%; left: 50%; -webkit-transform: translate3d(-50%, -50%, 0); -moz-transform: translate3d(-50%, -50%, 0); -o-transform: translate3d(-50%, -50%, 0); -ms-transform: translate3d(-50%, -50%, 0); transform: translate3d(-50%, -50%, 0) ; -webkit-perspective: 1200; -moz-perspective: 1200; -o-perspective: 1200; -ms-perspective: 1200; perspective: 1200 ; opacity: 0 } .flipper { position: relative; display: block; height: inherit; width: inherit; -webkit-animation: flip 1.2s infinite ease-in-out; -moz-animation: flip 1.2s infinite ease-in-out; -o-animation: flip 1.2s infinite ease-in-out; -ms-animation: flip 1.2s infinite ease-in-out; animation: flip 1.2s infinite ease-in-out ; -webkit-transform-style: preserve-3d; } .flipper .front, .flipper .back { - position: absolute; top: 0; left: 0; backface-visibility: hidden; /*transform-style: preserve-3d;*/ display: block; + position: absolute; top: 0; left: 0; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; -o-backface-visibility: hidden; -ms-backface-visibility: hidden; backface-visibility: hidden ; /*transform-style: preserve-3d;*/ display: block; background-color: #d50000; height: 100%; width: 100%; /*outline: 1px solid transparent; /* FF AA fix */ } .flipper .back { background-color: white; z-index: 800; -webkit-transform: rotateY(-180deg) ; -moz-transform: rotateY(-180deg) ; -o-transform: rotateY(-180deg) ; -ms-transform: rotateY(-180deg) ; transform: rotateY(-180deg) } diff --git a/src/Ui/template/wrapper.html b/src/Ui/template/wrapper.html index b2046d3e..049fdc2b 100644 --- a/src/Ui/template/wrapper.html +++ b/src/Ui/template/wrapper.html @@ -15,6 +15,7 @@ // If we are inside iframe escape from it if (window.self !== window.top) window.open(window.location.toString(), "_top"); if (window.self !== window.top) window.stop(); +if (window.self !== window.top && document.execCommand) document.execCommand("Stop", false) // Dont allow site to load in a popup if (window.opener) document.write("Opener not allowed") @@ -51,10 +52,13 @@ if (window.opener && window.stop) window.stop() - +