Rev957, Sidebar displays onion peers in graph, Sidebar display bad file retry number, Sidebar site Update/Pause/Delete, Ratelimit sidebar update, Encoded typo, Fix onion findHashId, More retry for bad files, Log file path errors, Testcase for self findhashIds, Testcase for Tor findHashId, Better Tor version parse, UiWebsocket callback on update/pause/resume/delete, Skip invalid postMessage messages

plugins/Sidebar/SidebarPlugin.py

@@ -65,14 +65,16 @@ class UiWebsocketPlugin(object):
         if peers_total:
             percent_connected = float(connected) / peers_total
             percent_connectable = float(connectable) / peers_total
+            percent_onion = float(onion) / peers_total
         else:
-            percent_connectable = percent_connected = 0
+            percent_connectable = percent_connected = percent_onion = 0
         body.append("""
             <li>
              <label>Peers</label>
              <ul class='graph'>
               <li style='width: 100%' class='total back-black' title="Total peers"></li>
               <li style='width: {percent_connectable:.0%}' class='connectable back-blue' title='Connectable peers'></li>
+              <li style='width: {percent_onion:.0%}' class='connected back-purple' title='Onion'></li>
               <li style='width: {percent_connected:.0%}' class='connected back-green' title='Connected peers'></li>
              </ul>
              <ul class='graph-legend'>
@@ -264,8 +266,8 @@ class UiWebsocketPlugin(object):
              <ul class='filelist'>
         """)
 
-        for bad_file in site.bad_files.keys():
+        for bad_file, tries in site.bad_files.iteritems():
-            body.append("""<li class='color-red' title="%s">%s</li>""" % (cgi.escape(bad_file, True), cgi.escape(bad_file, True)))
+            body.append("""<li class='color-red' title="%s (%s tries)">%s</li>""" % (cgi.escape(bad_file, True), tries, cgi.escape(bad_file, True)))
 
         body.append("""
              </ul>
@@ -296,6 +298,25 @@ class UiWebsocketPlugin(object):
             </li>
         """.format(**locals()))
 
+    def sidebarRenderControls(self, body, site):
+        auth_address = self.user.getAuthAddress(self.site.address)
+        if self.site.settings["serving"]:
+            class_pause = ""
+            class_resume = "hidden"
+        else:
+            class_pause = "hidden"
+            class_resume = ""
+
+        body.append("""
+            <li>
+             <label>Site control</label>
+             <a href='#Update' class='button noupdate' id='button-update'>Update</a>
+             <a href='#Pause' class='button {class_pause}' id='button-pause'>Pause</a>
+             <a href='#Resume' class='button {class_resume}' id='button-resume'>Resume</a>
+             <a href='#Delete' class='button noupdate' id='button-delete'>Delete</a>
+            </li>
+        """.format(**locals()))
+
     def sidebarRenderOwnedCheckbox(self, body, site):
         if self.site.settings["own"]:
             checked = "checked='checked'"
@@ -362,10 +383,11 @@ class UiWebsocketPlugin(object):
         has_optional = self.sidebarRenderOptionalFileStats(body, site)
         if has_optional:
             self.sidebarRenderOptionalFileSettings(body, site)
-        if site.bad_files:
-            self.sidebarRenderBadFiles(body, site)
         self.sidebarRenderDbOptions(body, site)
         self.sidebarRenderIdentity(body, site)
+        self.sidebarRenderControls(body, site)
+        if site.bad_files:
+            self.sidebarRenderBadFiles(body, site)
 
         self.sidebarRenderOwnedCheckbox(body, site)
         body.append("<div class='settings-owned'>")

plugins/Sidebar/media-globe/globe.js

@@ -433,4 +433,3 @@ DAT.Globe = function(container, opts) {
   return this;
 
 };
-

plugins/Sidebar/media/Sidebar.coffee

@@ -108,7 +108,8 @@ class Sidebar extends Class
 			@original_set_site_info.apply(wrapper, arguments)
 
 	setSiteInfo: (site_info) ->
-		@updateHtmlTag()
+		RateLimit 1000, =>
+			@updateHtmlTag()
 		@displayGlobe()
 
 
@@ -137,7 +138,7 @@ class Sidebar extends Class
 				@log "Patching content"
 				morphdom @tag.find(".content")[0], '<div class="content">'+res+'</div>', {
 					onBeforeMorphEl: (from_el, to_el) ->  # Ignore globe loaded state
-						if from_el.className == "globe"
+						if from_el.className == "globe" or from_el.className.indexOf("noupdate") >= 0
 							return false
 						else
 							return true
@@ -234,6 +235,34 @@ class Sidebar extends Class
 				@updateHtmlTag()
 			return false
 
+		# Update site
+		@tag.find("#button-update").off("click").on "click", =>
+			@tag.find("#button-update").addClass("loading")
+			wrapper.ws.cmd "siteUpdate", wrapper.site_info.address, =>
+				wrapper.notifications.add "done-updated", "done", "Site updated!", 5000
+				@tag.find("#button-update").removeClass("loading")
+			return false
+
+		# Pause site
+		@tag.find("#button-pause").off("click").on "click", =>
+			@tag.find("#button-pause").addClass("hidden")
+			wrapper.ws.cmd "sitePause", wrapper.site_info.address
+			return false
+
+		# Resume site
+		@tag.find("#button-resume").off("click").on "click", =>
+			@tag.find("#button-resume").addClass("hidden")
+			wrapper.ws.cmd "siteResume", wrapper.site_info.address
+			return false
+
+		# Delete site
+		@tag.find("#button-delete").off("click").on "click", =>
+			wrapper.displayConfirm "Are you sure?", "Delete this site", =>
+				@tag.find("#button-delete").addClass("loading")
+				wrapper.ws.cmd "siteDelete", wrapper.site_info.address, ->
+					document.location = $(".fixbutton-bg").attr("href")
+			return false
+
 		# Owned checkbox
 		@tag.find("#checkbox-owned").off("click").on "click", =>
 			wrapper.ws.cmd "siteSetOwned", [@tag.find("#checkbox-owned").is(":checked")]

plugins/Sidebar/media/Sidebar.css

@@ -13,8 +13,10 @@
 .sidebar { background-color: #212121; position: fixed; backface-visibility: hidden; right: -1200px; height: 100%; width: 1200px; } /*box-shadow: inset 0px 0px 10px #000*/
 .sidebar .content { margin: 30px; font-family: "Segoe UI Light", "Segoe UI", "Helvetica Neue"; color: white; width: 375px; height: 300px; font-weight: 200 }
 .sidebar h1, .sidebar h2 { font-weight: lighter; }
-.sidebar .button { margin: 0px; display: inline-block; }
+.sidebar .button { margin: 0px; display: inline-block; transition: all 0.3s; box-sizing: border-box; max-width: 160px }
-
+.sidebar .button.hidden { padding: 0px; max-width: 0px; opacity: 0; pointer-events: none }
+.sidebar #button-delete { background-color: transparent; border: 1px solid #333; color: #AAA; margin-left: 10px }
+.sidebar #button-delete:hover { border: 1px solid #666; color: white }
 
 /* FIELDS */
 
@@ -23,7 +25,7 @@
 .sidebar .fields > li:after, .sidebar .fields .settings-owned > li:after { clear: both; content: ''; display: block }
 .sidebar .fields label {
 	font-family: Consolas, monospace; text-transform: uppercase; font-size: 13px; color: #ACACAC; display: inline-block; margin-bottom: 10px;
-	vertical-align: text-bottom; margin-right: 10px;
+	vertical-align: text-bottom; margin-right: 10px; width: 100%
 }
 .sidebar .fields label small { font-weight: normal; color: white; text-transform: none; }
 .sidebar .fields .text { background-color: black; border: 0px; padding: 10px; color: white; border-radius: 3px; width: 250px; font-family: Consolas, monospace; }

plugins/Sidebar/media/all.css

@@ -67,8 +67,10 @@
 .sidebar { background-color: #212121; position: fixed; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; -o-backface-visibility: hidden; -ms-backface-visibility: hidden; backface-visibility: hidden ; right: -1200px; height: 100%; width: 1200px; } /*box-shadow: inset 0px 0px 10px #000*/
 .sidebar .content { margin: 30px; font-family: "Segoe UI Light", "Segoe UI", "Helvetica Neue"; color: white; width: 375px; height: 300px; font-weight: 200 }
 .sidebar h1, .sidebar h2 { font-weight: lighter; }
-.sidebar .button { margin: 0px; display: inline-block; }
+.sidebar .button { margin: 0px; display: inline-block; -webkit-transition: all 0.3s; -moz-transition: all 0.3s; -o-transition: all 0.3s; -ms-transition: all 0.3s; transition: all 0.3s ; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; -o-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box ; max-width: 160px }
-
+.sidebar .button.hidden { padding: 0px; max-width: 0px; opacity: 0; pointer-events: none }
+.sidebar #button-delete { background-color: transparent; border: 1px solid #333; color: #AAA; margin-left: 10px }
+.sidebar #button-delete:hover { border: 1px solid #666; color: white }
 
 /* FIELDS */
 
@@ -77,7 +79,7 @@
 .sidebar .fields > li:after, .sidebar .fields .settings-owned > li:after { clear: both; content: ''; display: block }
 .sidebar .fields label {
 	font-family: Consolas, monospace; text-transform: uppercase; font-size: 13px; color: #ACACAC; display: inline-block; margin-bottom: 10px;
-	vertical-align: text-bottom; margin-right: 10px;
+	vertical-align: text-bottom; margin-right: 10px; width: 100%
 }
 .sidebar .fields label small { font-weight: normal; color: white; text-transform: none; }
 .sidebar .fields .text { background-color: black; border: 0px; padding: 10px; color: white; -webkit-border-radius: 3px; -moz-border-radius: 3px; -o-border-radius: 3px; -ms-border-radius: 3px; border-radius: 3px ; width: 250px; font-family: Consolas, monospace; }

plugins/Sidebar/media/all.js

@@ -86,7 +86,6 @@
 }).call(this);
 
 
-
 /* ---- plugins/Sidebar/media/Scrollable.js ---- */
 
 
@@ -316,7 +315,11 @@ window.initScrollable = function () {
     };
 
     Sidebar.prototype.setSiteInfo = function(site_info) {
-      this.updateHtmlTag();
+      RateLimit(1000, (function(_this) {
+        return function() {
+          return _this.updateHtmlTag();
+        };
+      })(this));
       return this.displayGlobe();
     };
 
@@ -341,7 +344,7 @@ window.initScrollable = function () {
             _this.log("Patching content");
             morphdom(_this.tag.find(".content")[0], '<div class="content">' + res + '</div>', {
               onBeforeMorphEl: function(from_el, to_el) {
-                if (from_el.className === "globe") {
+                if (from_el.className === "globe" || from_el.className.indexOf("noupdate") >= 0) {
                   return false;
                 } else {
                   return true;
@@ -449,6 +452,41 @@ window.initScrollable = function () {
           return false;
         };
       })(this));
+      this.tag.find("#button-update").off("click").on("click", (function(_this) {
+        return function() {
+          _this.tag.find("#button-update").addClass("loading");
+          wrapper.ws.cmd("siteUpdate", wrapper.site_info.address, function() {
+            wrapper.notifications.add("done-updated", "done", "Site updated!", 5000);
+            return _this.tag.find("#button-update").removeClass("loading");
+          });
+          return false;
+        };
+      })(this));
+      this.tag.find("#button-pause").off("click").on("click", (function(_this) {
+        return function() {
+          _this.tag.find("#button-pause").addClass("hidden");
+          wrapper.ws.cmd("sitePause", wrapper.site_info.address);
+          return false;
+        };
+      })(this));
+      this.tag.find("#button-resume").off("click").on("click", (function(_this) {
+        return function() {
+          _this.tag.find("#button-resume").addClass("hidden");
+          wrapper.ws.cmd("siteResume", wrapper.site_info.address);
+          return false;
+        };
+      })(this));
+      this.tag.find("#button-delete").off("click").on("click", (function(_this) {
+        return function() {
+          wrapper.displayConfirm("Are you sure?", "Delete this site", function() {
+            _this.tag.find("#button-delete").addClass("loading");
+            return wrapper.ws.cmd("siteDelete", wrapper.site_info.address, function() {
+              return document.location = $(".fixbutton-bg").attr("href");
+            });
+          });
+          return false;
+        };
+      })(this));
       this.tag.find("#checkbox-owned").off("click").on("click", (function(_this) {
         return function() {
           return wrapper.ws.cmd("siteSetOwned", [_this.tag.find("#checkbox-owned").is(":checked")]);

src/Config.py

@@ -8,7 +8,7 @@ class Config(object):
 
     def __init__(self, argv):
         self.version = "0.3.6"
-        self.rev = 949
+        self.rev = 957
         self.argv = argv
         self.action = None
         self.config_file = "zeronet.conf"

src/Content/ContentManager.py

@@ -311,7 +311,7 @@ class ContentManager(object):
                 ignored = True
             elif not re.match("^[a-zA-Z0-9_\.\+\-/]+$", file_relative_path):
                 ignored = True
-                self.log.error("- [ERROR] Only ascii encodes filenames allowed: %s" % file_relative_path)
+                self.log.error("- [ERROR] Only ascii encoded filenames allowed: %s" % file_relative_path)
             elif optional_pattern and re.match(optional_pattern, file_relative_path):
                 optional = True
 

src/File/FileRequest.py

@@ -306,25 +306,35 @@ class FileRequest(object):
 
         found = site.worker_manager.findOptionalHashIds(params["hash_ids"])
 
-        back = {}
+        back_ip4 = {}
+        back_onion = {}
         for hash_id, peers in found.iteritems():
-            back[hash_id] = [helper.packAddress(peer.ip, peer.port) for peer in peers]
+            back_onion[hash_id] = [helper.packOnionAddress(peer.ip, peer.port) for peer in peers if peer.ip.endswith("onion")]
+            back_ip4[hash_id] = [helper.packAddress(peer.ip, peer.port) for peer in peers if not peer.ip.endswith("onion")]
+
         # Check my hashfield
-        if config.ip_external:
+        if self.server.tor_manager and self.server.tor_manager.site_onions.get(site.address):  # Running onion
-            my_ip = config.ip_external
+            my_ip = helper.packOnionAddress(self.server.tor_manager.site_onions[site.address], self.server.port)
-        else:
+            my_back = back_onion
-            my_ip = self.server.ip
+        elif config.ip_external:  # External ip definied
+            my_ip = helper.packAddress(config.ip_external, self.server.port)
+            my_back = back_ip4
+        else:  # No external ip defined
+            my_ip = my_ip = helper.packAddress(self.server.ip, self.server.port)
+            my_back = back_ip4
+
         for hash_id in params["hash_ids"]:
             if hash_id in site.content_manager.hashfield:
-                if hash_id not in back:
+                if hash_id not in my_back:
-                    back[hash_id] = []
+                    my_back[hash_id] = []
-                back[hash_id].append(helper.packAddress(my_ip, self.server.port))  # Add myself
+                my_back[hash_id].append(my_ip)  # Add myself
+
         if config.verbose:
             self.log.debug(
-                "Found: %s/%s" %
+                "Found: %s,%s/%s" %
-                (len(back), len(params["hash_ids"]))
+                (len(back_ip4), len(back_onion), len(params["hash_ids"]))
             )
-        self.response({"peers": back})
+        self.response({"peers": back_ip4, "peers_onion": back_onion})
 
     def actionSetHashfield(self, params):
         site = self.sites.get(params["site"])

src/Peer/Peer.py

@@ -274,7 +274,15 @@ class Peer(object):
         res = self.request("findHashIds", {"site": self.site.address, "hash_ids": hash_ids})
         if not res or "error" in res:
             return False
-        return {key: map(helper.unpackAddress, val) for key, val in res["peers"].iteritems()}
+        # Unpack IP4
+        back = {key: map(helper.unpackAddress, val) for key, val in res["peers"].iteritems()}
+        # Unpack onion
+        for hash, onion_peers in res.get("peers_onion", {}).iteritems():
+            if not hash in back:
+                back[hash] = []
+            back[hash] += map(helper.unpackOnionAddress, onion_peers)
+
+        return back
 
     # Send my hashfield to peer
     # Return: True if sent

src/Site/Site.py

@@ -176,7 +176,7 @@ class Site(object):
     # Retry download bad files
     def retryBadFiles(self, force=False):
         for bad_file, tries in self.bad_files.items():
-            if force or random.randint(0, min(20, tries)) == 0:  # Larger number tries = less likely to check every 15min
+            if force or random.randint(0, min(40, tries)) < 4:  # Larger number tries = less likely to check every 15min
                 self.needFile(bad_file, update=True, blocking=False)
 
     # Download all files of the site

src/Site/SiteStorage.py

@@ -247,6 +247,7 @@ class SiteStorage:
 
         file_abspath = os.path.dirname(os.path.abspath(file_path))
         if ".." in file_path or not file_abspath.startswith(self.allowed_dir):
+            self.site.log.error(u"File %s not in allowed dir: %s" % (file_path, self.allowed_dir))
             raise Exception(u"File not allowed: %s" % file_path)
         return file_path
 

src/Test/TestPeer.py

@@ -155,3 +155,10 @@ class TestPeer:
             1234: [('1.2.3.4', 1544), ('1.2.3.5', 1545)],
             1235: [('1.2.3.5', 1545), ('1.2.3.6', 1546)]
         }
+
+        # Test my address adding
+        site.content_manager.hashfield.append(1234)
+
+        res = peer_file_server.findHashIds([1234, 1235])
+        assert res[1234] == [('1.2.3.4', 1544), ('1.2.3.5', 1545), ("127.0.0.1", 1544)]
+        assert res[1235] == [('1.2.3.5', 1545), ('1.2.3.6', 1546)]

src/Test/TestTor.py

@@ -102,6 +102,43 @@ class TestTor:
         assert peer_source.pex(need_num=10) == 1  # Need >5 to return also return non-connected peers
         assert "bka4ht2bzxchy44r.onion:1555" in site_temp.peers
 
+    def testFindHash(self, tor_manager, file_server, site, site_temp):
+        file_server.ip_incoming = {}  # Reset flood protection
+        file_server.sites[site.address] = site
+        file_server.tor_manager = tor_manager
+
+        client = FileServer("127.0.0.1", 1545)
+        client.sites[site_temp.address] = site_temp
+        site_temp.connection_server = client
+
+        # Add file_server as peer to client
+        peer_file_server = site_temp.addPeer("127.0.0.1", 1544)
+
+        assert peer_file_server.findHashIds([1234]) == {}
+
+        # Add fake peer with requred hash
+        fake_peer_1 = site.addPeer("bka4ht2bzxchy44r.onion", 1544)
+        fake_peer_1.hashfield.append(1234)
+        fake_peer_2 = site.addPeer("1.2.3.5", 1545)
+        fake_peer_2.hashfield.append(1234)
+        fake_peer_2.hashfield.append(1235)
+        fake_peer_3 = site.addPeer("1.2.3.6", 1546)
+        fake_peer_3.hashfield.append(1235)
+        fake_peer_3.hashfield.append(1236)
+
+        assert peer_file_server.findHashIds([1234, 1235]) == {
+            1234: [('1.2.3.5', 1545), ("bka4ht2bzxchy44r.onion", 1544)],
+            1235: [('1.2.3.6', 1546), ('1.2.3.5', 1545)]
+        }
+
+        # Test my address adding
+        site.content_manager.hashfield.append(1234)
+        my_onion_address = tor_manager.getOnion(site_temp.address)+".onion"
+
+        res = peer_file_server.findHashIds([1234, 1235])
+        assert res[1234] == [('1.2.3.5', 1545), ("bka4ht2bzxchy44r.onion", 1544), (my_onion_address, 1544)]
+        assert res[1235] == [('1.2.3.6', 1546), ('1.2.3.5', 1545)]
+
     def testSiteOnion(self, tor_manager):
         assert tor_manager.getOnion("address1") != tor_manager.getOnion("address2")
         assert tor_manager.getOnion("address1") == tor_manager.getOnion("address1")

src/Tor/TorManager.py

@@ -153,7 +153,7 @@ class TorManager:
 
                 version = re.search('Tor="([0-9\.]+)', res_protocol).group(1)
                 # Version 0.2.7.5 required because ADD_ONION support
-                assert int(version.replace(".", "0")) >= 20705, "Tor version >=0.2.7.5 required"
+                assert float(version.replace(".", "0", 2)) >= 207.5, "Tor version >=0.2.7.5 required, found: %s" % version
 
                 # Auth cookie file
                 cookie_match = re.search('COOKIEFILE="(.*?)"', res_protocol)

src/Ui/UiWebsocket.py

@@ -534,9 +534,13 @@ class UiWebsocket(object):
 
     # Update site content.json
     def actionSiteUpdate(self, to, address):
+        def updateThread():
+            site.update()
+            self.response(to, "Updated")
+
         site = self.server.sites.get(address)
         if site and (site.address == self.site.address or "ADMIN" in self.site.settings["permissions"]):
-            gevent.spawn(site.update)
+            gevent.spawn(updateThread)
         else:
             self.response(to, {"error": "Unknown site: %s" % address})
 
@@ -548,6 +552,7 @@ class UiWebsocket(object):
             site.saveSettings()
             site.updateWebsocket()
             site.worker_manager.stopWorkers()
+            self.response(to, "Paused")
         else:
             self.response(to, {"error": "Unknown site: %s" % address})
 
@@ -560,6 +565,7 @@ class UiWebsocket(object):
             gevent.spawn(site.update, announce=True)
             time.sleep(0.001)  # Wait for update thread starting
             site.updateWebsocket()
+            self.response(to, "Resumed")
         else:
             self.response(to, {"error": "Unknown site: %s" % address})
 
@@ -574,6 +580,7 @@ class UiWebsocket(object):
             site.updateWebsocket()
             SiteManager.site_manager.delete(address)
             self.user.deleteSiteData(address)
+            self.response(to, "Deleted")
         else:
             self.response(to, {"error": "Unknown site: %s" % address})
 

src/Ui/media/Wrapper.coffee

@@ -79,6 +79,9 @@ class Wrapper
 				@opener = false
 
 		message = e.data
+		if not message.cmd
+			return false
+
 		if window.postmessage_nonce_security and message.wrapper_nonce != window.wrapper_nonce
 			@log "Message nonce error:", message.wrapper_nonce, '!=', window.wrapper_nonce
 			@actionNotification({"params": ["error", "Message wrapper_nonce error, please report!"]})

src/Ui/media/all.js

@@ -857,6 +857,9 @@ jQuery.extend( jQuery.easing,
         }
       }
       message = e.data;
+      if (!message.cmd) {
+        return false;
+      }
       if (window.postmessage_nonce_security && message.wrapper_nonce !== window.wrapper_nonce) {
         this.log("Message nonce error:", message.wrapper_nonce, '!=', window.wrapper_nonce);
         this.actionNotification({