Rev390, Fix sidebar error on non locatable IPs, Configurable bootstrap torrent trackers, Multi-line config file settings, Evenly distributed tracker announce to work better on passive connections, Avoid iframe sandbox escape by using nonces, Better html error messages, Display proper error on invalid startup parameters

plugins/Sidebar/SidebarPlugin.py

@@ -376,7 +376,7 @@ class UiWebsocketPlugin(object):
                 else:
                     loc = geodb.get(peer.ip)
                     loc_cache[peer.ip] = loc
-                if not loc:
+                if not loc or "location" not in loc:
                     continue
 
                 # Create position array

src/Config.py

@@ -8,7 +8,7 @@ class Config(object):
 
     def __init__(self, argv):
         self.version = "0.3.2"
-        self.rev = 378
+        self.rev = 390
         self.argv = argv
         self.action = None
         self.createParser()
@@ -29,6 +29,14 @@ class Config(object):
 
     # Create command line arguments
     def createArguments(self):
+        trackers = [
+            "udp://open.demonii.com:1337",
+            "udp://tracker.leechers-paradise.org:6969",
+            "udp://9.rarbg.com:2710",
+            "http://tracker.aletorrenty.pl:2710/announce",
+            "http://retracker.telecom.kz/announce",
+            "http://torrent.gresille.org/announce"
+        ]
         # Platform specific
         if sys.platform.startswith("win"):
             coffeescript = "type %s | tools\\coffee\\coffee.cmd"
@@ -122,7 +130,8 @@ class Config(object):
         self.parser.add_argument('--fileserver_port', help='FileServer bind port', default=15441, type=int, metavar='port')
         self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true')
         self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port')
-        self.parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip')
+        self.parser.add_argument('--ip_external', help='Set reported external ip (tested on start if None)', metavar='ip')
+        self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*')
         self.parser.add_argument('--use_openssl', help='Use OpenSSL liblary for speedup',
                                  type='bool', choices=[True, False], default=use_openssl)
         self.parser.add_argument('--disable_encryption', help='Disable connection encryption', action='store_true')
@@ -238,7 +247,8 @@ class Config(object):
                     if section != "global":  # If not global prefix key with section
                         key = section + "_" + key
                     if val:
-                        argv.insert(1, val)
+                        for line in val.strip().split("\n"): # Allow multi-line values
+                            argv.insert(1, line)
                     argv.insert(1, "--%s" % key)
         return argv
 

src/File/FileServer.py

@@ -170,11 +170,15 @@ class FileServer(ConnectionServer):
     # Announce sites every 20 min
     def announceSites(self):
         import gc
+        first_announce = True  # First start
         while 1:
-            time.sleep(20 * 60)  # Announce sites every 20 min
+            # Sites healthcare
             for address, site in self.sites.items():
                 if site.settings["serving"]:
-                    site.announce()  # Announce site to tracker
+                    if first_announce:  # Announce to all trackers on startup
+                        site.announce()
+                    else:  # If not first run only use PEX
+                        site.announcePex()
 
                     # Reset bad file retry counter
                     for inner_path in site.bad_files:
@@ -195,6 +199,20 @@ class FileServer(ConnectionServer):
             site = None
             gc.collect()  # Implicit grabage collection
 
+            # Find new peers
+            for tracker_i in range(len(config.trackers)):
+                time.sleep(60 * 20 / len(config.trackers))  # Query all trackers one-by-one in 20 minutes evenly distributed
+
+                for address, site in self.sites.items():
+                    site.announce(num = 1, pex = False)
+                    time.sleep(2)
+
+            first_announce = False
+
+
+
+
+
     # Detects if computer back from wakeup
     def wakeupWatcher(self):
         last_time = time.time()

src/Site/Site.py

@@ -38,6 +38,7 @@ class Site:
         self.peers = {}  # Key: ip:port, Value: Peer.Peer
         self.peer_blacklist = SiteManager.peer_blacklist  # Ignore this peers (eg. myself)
         self.last_announce = 0  # Last announce time to tracker
+        self.last_tracker_id = random.randint(0, 10)  # Last announced tracker id
         self.worker_manager = WorkerManager(self)  # Handle site download from other peers
         self.bad_files = {}  # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
         self.content_updated = None  # Content.js update time
@@ -510,12 +511,13 @@ class Site:
 
     # Gather peers from tracker
     # Return: Complete time or False on error
-    def announceTracker(self, protocol, ip, port, fileserver_port, address_hash, my_peer_id):
+    def announceTracker(self, protocol, address, fileserver_port, address_hash, my_peer_id):
         s = time.time()
         if protocol == "udp":  # Udp tracker
             if config.disable_udp:
                 return False  # No udp supported
-            tracker = UdpTrackerClient(ip, port)
+            ip, port = address.split(":")
+            tracker = UdpTrackerClient(ip, int(port))
             tracker.peer_port = fileserver_port
             try:
                 tracker.connect()
@@ -535,7 +537,7 @@ class Site:
             }
             req = None
             try:
-                url = "http://" + ip + "?" + urllib.urlencode(params)
+                url = "http://" + address + "?" + urllib.urlencode(params)
                 # Load url
                 with gevent.Timeout(10, False):  # Make sure of timeout
                     req = urllib2.urlopen(url, timeout=8)
@@ -577,10 +579,17 @@ class Site:
         return time.time() - s
 
     # Add myself and get other peers from tracker
-    def announce(self, force=False):
+    def announce(self, force=False, num=5, pex=True):
         if time.time() < self.last_announce + 30 and not force:
             return  # No reannouncing within 30 secs
         self.last_announce = time.time()
+
+        trackers = config.trackers
+        if num == 1:  # Only announce on one tracker, increment the queried tracker id
+            self.last_tracker_id += 1
+            self.last_tracker_id = self.last_tracker_id % len(trackers)
+            trackers = [trackers[self.last_tracker_id]]  # We only going to use this one
+
         errors = []
         slow = []
         address_hash = hashlib.sha1(self.address).hexdigest()  # Site address hash
@@ -595,39 +604,42 @@ class Site:
         announced = 0
         threads = []
 
-        for protocol, ip, port in SiteManager.TRACKERS:  # Start announce threads
+        for tracker in trackers:  # Start announce threads
-            thread = gevent.spawn(self.announceTracker, protocol, ip, port, fileserver_port, address_hash, my_peer_id)
+            protocol, address = tracker.split("://")
+            thread = gevent.spawn(self.announceTracker, protocol, address, fileserver_port, address_hash, my_peer_id)
             threads.append(thread)
-            thread.ip = ip
+            thread.address = address
             thread.protocol = protocol
+            if len(threads) > num: break  # Announce limit
 
         gevent.joinall(threads)  # Wait for announce finish
 
         for thread in threads:
             if thread.value:
                 if thread.value > 1:
-                    slow.append("%.2fs %s://%s" % (thread.value, thread.protocol, thread.ip))
+                    slow.append("%.2fs %s://%s" % (thread.value, thread.protocol, thread.address))
                 announced += 1
             else:
-                errors.append("%s://%s" % (thread.protocol, thread.ip))
+                errors.append("%s://%s" % (thread.protocol, thread.address))
 
         # Save peers num
         self.settings["peers"] = len(self.peers)
         self.saveSettings()
 
-        if len(errors) < len(SiteManager.TRACKERS):  # Less errors than total tracker nums
+        if len(errors) < min(num, len(trackers)):  # Less errors than total tracker nums
             self.log.debug(
                 "Announced port %s to %s trackers in %.3fs, errors: %s, slow: %s" %
                 (fileserver_port, announced, time.time() - s, errors, slow)
             )
         else:
-            self.log.error("Announced to %s trackers in %.3fs, failed" % (announced, time.time() - s))
+            self.log.error("Announce to %s trackers in %.3fs, failed" % (announced, time.time() - s))
 
-        if not [peer for peer in self.peers.values() if peer.connection and peer.connection.connected]:
+        if pex:
-            # If no connected peer yet then wait for connections
+            if not [peer for peer in self.peers.values() if peer.connection and peer.connection.connected]:
-            gevent.spawn_later(3, self.announcePex, need_num=10)  # Spawn 3 secs later
+                # If no connected peer yet then wait for connections
-        else:  # Else announce immediately
+                gevent.spawn_later(3, self.announcePex, need_num=10)  # Spawn 3 secs later
-            self.announcePex()
+            else:  # Else announce immediately
+                self.announcePex()
 
     # Keep connections to get the updates (required for passive clients)
     def needConnections(self, num=3):

src/Site/SiteManager.py

@@ -6,26 +6,6 @@ import os
 from Plugin import PluginManager
 from Config import config
 
-TRACKERS = [
-    ("udp", "open.demonii.com", 1337),
-    # ("udp", "sugoi.pomf.se", 2710),
-    # ("udp", "tracker.coppersurfer.tk", 80),
-    ("udp", "tracker.leechers-paradise.org", 6969),
-    ("udp", "9.rarbg.com", 2710),
-    # ("udp", "www.eddie4.nl", 6969),
-    # ("udp", "trackr.sytes.net", 80),
-    # ("udp", "tracker4.piratux.com", 6969)
-    # ("http", "exodus.desync.com:80/announce", None), Off
-    ("http", "tracker.aletorrenty.pl:2710/announce", None),
-    # ("http", "torrent.gresille.org/announce", None), # Slow
-    # ("http", "announce.torrentsmd.com:6969/announce", None), # Off
-    # ("http", "i.bandito.org/announce", None), # Off
-    ("http", "retracker.telecom.kz/announce", None),
-    ("http", "torrent.gresille.org/announce", None),
-
-]
-
-
 @PluginManager.acceptPlugins
 class SiteManager(object):
 

src/Ui/UiRequest.py

@@ -4,6 +4,8 @@ import os
 import mimetypes
 import json
 import cgi
+import string
+import random
 
 from Config import config
 from Site import SiteManager
@@ -69,7 +71,7 @@ class UiRequest(object):
             return self.actionConsole()
         # Site media wrapper
         else:
-            if self.get.get("wrapper") == "False":
+            if self.get.get("wrapper_nonce"):
                 return self.actionSiteMedia("/media" + path)  # Only serve html files with frame
             else:
                 body = self.actionWrapper(path)
@@ -202,7 +204,6 @@ class UiRequest(object):
         else:  # Bad url
             return False
 
-
     def renderWrapper(self, site, path, inner_path, title, extra_headers):
         file_inner_path = inner_path
         if not file_inner_path:
@@ -219,10 +220,12 @@ class UiRequest(object):
         body_style = ""
         meta_tags = ""
 
+        wrapper_nonce = self.getWrapperNonce()
+
         if self.env.get("QUERY_STRING"):
-            query_string = "?" + self.env["QUERY_STRING"] + "&wrapper=False"
+            query_string = "?%s&wrapper_nonce=%s" % (self.env["QUERY_STRING"], wrapper_nonce)
         else:
-            query_string = "?wrapper=False"
+            query_string = "?wrapper_nonce=%s" % wrapper_nonce
 
         if self.isProxyRequest():  # Its a remote proxy request
             if self.env["REMOTE_ADDR"] == "127.0.0.1":  # Local client, the server address also should be 127.0.0.1
@@ -260,6 +263,13 @@ class UiRequest(object):
             homepage=homepage
         )
 
+    # Create a new wrapper nonce that allows to get one html file without the wrapper
+    def getWrapperNonce(self):
+        wrapper_nonce = ''.join(
+            random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24)
+        )
+        self.server.wrapper_nonces.append(wrapper_nonce)
+        return wrapper_nonce
 
     # Returns if media request allowed from that referer
     def isMediaRequestAllowed(self, site_address, referer):
@@ -276,6 +286,14 @@ class UiRequest(object):
 
         match = re.match("/media/(?P<address>[A-Za-z0-9\._-]+)/(?P<inner_path>.*)", path)
 
+        # Check wrapper nonce
+        content_type = self.getContentType(path)
+        if "htm" in content_type:  # Valid nonce must present to render html files
+            wrapper_nonce = self.get["wrapper_nonce"]
+            if wrapper_nonce not in self.server.wrapper_nonces:
+                return self.error403("Wrapper nonce error.")
+            self.server.wrapper_nonces.remove(self.get["wrapper_nonce"])
+
         referer = self.env.get("HTTP_REFERER")
         if referer and match:  # Only allow same site to receive media
             if not self.isMediaRequestAllowed(match.group("address"), referer):
@@ -421,24 +439,38 @@ class UiRequest(object):
     # - Errors -
 
     # Send bad request error
-    def error400(self):
+    def error400(self, message=""):
         self.sendHeader(400)
-        return "Bad Request"
+        return self.formatError("Bad Request", message)
 
     # You are not allowed to access this
-    def error403(self, message="Forbidden"):
+    def error403(self, message=""):
         self.sendHeader(403)
-        return message
+        return self.formatError("Forbidden", message)
 
     # Send file not found error
-    def error404(self, path=None):
+    def error404(self, path=""):
         self.sendHeader(404)
-        return "Not Found: %s" % path.encode("utf8")
+        return self.formatError("Not Found", path.encode("utf8"))
 
     # Internal server error
     def error500(self, message=":("):
         self.sendHeader(500)
-        return "<h1>Server error</h1>%s" % cgi.escape(message)
+        return self.formatError("Server error", cgi.escape(message))
+
+    def formatError(self, title, message):
+        details = {key: val for key, val in self.env.items() if hasattr(val, "endswith") and "COOKIE" not in key }
+        return """
+            <h1>%s</h1>
+            <h2>%s</h3>
+            <h3>Please <a href="https://github.com/HelloZeroNet/ZeroNet/issues">report it</a> if you think this an error.</h3>
+            <h4>Details:</h4>
+            <pre>%s</pre>
+            <style>
+            * { font-family: Consolas, Monospace; color: #333 }
+            pre { padding: 10px; background-color: #EEE }
+            </style>
+        """ % (title, message, json.dumps(details, indent=4))
 
 
 # - Reload for eaiser developing -

src/Ui/UiServer.py

@@ -55,6 +55,7 @@ class UiServer:
         self.port = config.ui_port
         if self.ip == "*":
             self.ip = ""  # Bind all
+        self.wrapper_nonces = []
         self.sites = SiteManager.site_manager.list()
         self.log = logging.getLogger(__name__)
 

src/Ui/media/all.css

@@ -35,7 +35,7 @@ a { color: black }
 	display: block; width: 80px; height: 80px; -webkit-transition: background-color 0.2s, box-shadow 0.5s; -moz-transition: background-color 0.2s, box-shadow 0.5s; -o-transition: background-color 0.2s, box-shadow 0.5s; -ms-transition: background-color 0.2s, box-shadow 0.5s; transition: background-color 0.2s, box-shadow 0.5s ; -webkit-transform: scale(0.6); -moz-transform: scale(0.6); -o-transform: scale(0.6); -ms-transform: scale(0.6); transform: scale(0.6) ; margin-left: -20px; margin-top: -20px; /* 2x size to prevent blur on anim */
 	/*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* -webkit-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -moz-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -o-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -ms-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1) ; */
 }
-.fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px }
+.fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; -o-backface-visibility: hidden; -ms-backface-visibility: hidden; backface-visibility: hidden ; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px }
 .fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px }
 .fixbutton-bg:hover { background-color: #AF3BFF }
 .fixbutton-bg:active { background-color: #9E2FEA; top: 1px; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none  }
@@ -45,7 +45,7 @@ a { color: black }
 
 .notifications { position: absolute; top: 0px; right: 80px; display: inline-block; z-index: 999; white-space: nowrap }
 .notification {
-	position: relative; float: right; clear: both; margin: 10px; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; -o-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box ; overflow: hidden; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; padding-bottom: 5px;
+	position: relative; float: right; clear: both; margin: 10px; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; -o-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box ; overflow: hidden; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; -o-backface-visibility: hidden; -ms-backface-visibility: hidden; backface-visibility: hidden ; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; padding-bottom: 5px;
 	color: #4F4F4F; font-family: 'Lucida Grande', 'Segoe UI', Helvetica, Arial, sans-serif; font-size: 14px; line-height: 20px; /*border: 1px solid rgba(210, 206, 205, 0.2)*/
 }
 .notification-icon {
@@ -112,7 +112,7 @@ a { color: black }
 .flipper-container { width: 40px; height: 40px; position: absolute; top: 0%; left: 50%; -webkit-transform: translate3d(-50%, -50%, 0); -moz-transform: translate3d(-50%, -50%, 0); -o-transform: translate3d(-50%, -50%, 0); -ms-transform: translate3d(-50%, -50%, 0); transform: translate3d(-50%, -50%, 0) ; -webkit-perspective: 1200; -moz-perspective: 1200; -o-perspective: 1200; -ms-perspective: 1200; perspective: 1200 ; opacity: 0 }
 .flipper { position: relative; display: block; height: inherit; width: inherit; -webkit-animation: flip 1.2s infinite ease-in-out; -moz-animation: flip 1.2s infinite ease-in-out; -o-animation: flip 1.2s infinite ease-in-out; -ms-animation: flip 1.2s infinite ease-in-out; animation: flip 1.2s infinite ease-in-out ; -webkit-transform-style: preserve-3d; }
 .flipper .front, .flipper .back {
-	position: absolute; top: 0; left: 0; backface-visibility: hidden; /*transform-style: preserve-3d;*/ display: block;
+	position: absolute; top: 0; left: 0; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; -o-backface-visibility: hidden; -ms-backface-visibility: hidden; backface-visibility: hidden ; /*transform-style: preserve-3d;*/ display: block;
 	background-color: #d50000; height: 100%; width: 100%; /*outline: 1px solid transparent; /* FF AA fix */
 }
 .flipper .back { background-color: white; z-index: 800; -webkit-transform: rotateY(-180deg) ; -moz-transform: rotateY(-180deg) ; -o-transform: rotateY(-180deg) ; -ms-transform: rotateY(-180deg) ; transform: rotateY(-180deg)  }

src/Ui/template/wrapper.html

@@ -15,6 +15,7 @@
 // If we are inside iframe escape from it
 if (window.self !== window.top) window.open(window.location.toString(), "_top");
 if (window.self !== window.top) window.stop();
+if (window.self !== window.top && document.execCommand) document.execCommand("Stop", false)
 
 // Dont allow site to load in a popup
 if (window.opener) document.write("Opener not allowed")
@@ -51,10 +52,13 @@ if (window.opener && window.stop) window.stop()
 
 
 <!-- Site Iframe -->
-<iframe src='{file_url}{query_string}' id='inner-iframe' sandbox="allow-forms allow-scripts allow-top-navigation allow-popups"></iframe>
+<iframe src='about:blank' id='inner-iframe' sandbox="allow-forms allow-scripts allow-top-navigation allow-popups"></iframe>
 
 <!-- Site info -->
 <script>
+console.log("Setting iframe src", document.getElementById("inner-iframe").src, "{file_url}{query_string}")
+document.getElementById("inner-iframe").src = "about:blank"
+document.getElementById("inner-iframe").src = "{file_url}{query_string}"
 address = "{address}"
 wrapper_key = "{wrapper_key}"
 file_inner_path = "{file_inner_path}"

src/Worker/WorkerManager.py

@@ -61,7 +61,7 @@ class WorkerManager:
                 elif (task["time_started"] and time.time() >= task["time_started"] + 15) or not self.workers:
                     # Task started more than 15 sec ago or no workers
                     self.log.debug("Task taking more than 15 secs, find more peers: %s" % task["inner_path"])
-                    task["site"].announce()  # Find more peers
+                    task["site"].announce(num=1)  # Find more peers
                     if task["peers"]:  # Release the peer lock
                         self.log.debug("Task peer lock release: %s" % task["inner_path"])
                         task["peers"] = []

src/main.py

@@ -14,6 +14,8 @@ update_after_shutdown = False  # If set True then update and restart zeronet aft
 # Load config
 from Config import config
 config.parse(silent=True)  # Plugins need to access the configuration
+if not config.arguments:  # Config parse failed, show the help screen and exit
+    config.parse()
 
 # Create necessary files and dirs
 if not os.path.isdir(config.log_dir):