Rev465, Display memory dump only in debug mode, Use sys.exit in trayicon, Optional files hashing, List function in SiteStorage, Test signing content, Test sign optional files, Test site storage

plugins/Stats/StatsPlugin.py

@@ -147,8 +147,13 @@ class UiRequestPlugin(object):
             yield "<br></td></tr>"
         yield "</table>"
 
+        # No more if not in debug mode
+        if not config.debug:
+            raise StopIteration
+
         # Object types
 
+
         obj_count = {}
         for obj in gc.get_objects():
             obj_type = str(type(obj))
@@ -250,10 +255,17 @@ class UiRequestPlugin(object):
         yield "Done in %.1f" % (time.time() - s)
 
     def actionDumpobj(self):
+
         import gc
         import sys
 
         self.sendHeader()
+
+        # No more if not in debug mode
+        if not config.debug:
+            yield "Not in debug mode"
+            raise StopIteration
+
         class_filter = self.get.get("class")
 
         yield """
@@ -276,10 +288,17 @@ class UiRequestPlugin(object):
         gc.collect()  # Implicit grabage collection
 
     def actionListobj(self):
+
         import gc
         import sys
 
         self.sendHeader()
+
+        # No more if not in debug mode
+        if not config.debug:
+            yield "Not in debug mode"
+            raise StopIteration
+
         type_filter = self.get.get("type")
 
         yield """

plugins/Trayicon/TrayiconPlugin.py

@@ -63,9 +63,9 @@ class ActionsPlugin(object):
     def quit(self):
         self.icon.die()
         time.sleep(0.1)
-        self.main.ui_server.stop()
+        sys.exit()
-        self.main.file_server.stop()
+        #self.main.ui_server.stop()
-        # sys.exit()
+        #self.main.file_server.stop()
 
     def opensite(self, url):
         import webbrowser

src/Config.py

@@ -8,7 +8,7 @@ class Config(object):
 
     def __init__(self, argv):
         self.version = "0.3.2"
-        self.rev = 452
+        self.rev = 465
         self.argv = argv
         self.action = None
         self.createParser()

src/Content/ContentManager.py

@@ -237,6 +237,37 @@ class ContentManager(object):
 
         return rules
 
+    # Hash files in directory
+    def hashFiles(self, dir_inner_path, ignore_pattern=None, optional_pattern=None):
+        files_node = {}
+        files_optional_node = {}
+
+        for file_relative_path in self.site.storage.list(dir_inner_path):
+            file_name = self.toFilename(file_relative_path)
+
+            ignored = optional = False
+            if file_name == "content.json":
+                ignored = True
+            elif ignore_pattern and re.match(ignore_pattern, file_relative_path):
+                ignored = True
+            elif file_name.startswith("."):
+                ignored = True
+            elif optional_pattern and re.match(optional_pattern, file_relative_path):
+                optional = True
+
+            if ignored:  # Ignore content.json, definied regexp and files starting with .
+                self.log.info("- [SKIPPED] %s" % file_relative_path)
+            else:
+                file_path = self.site.storage.getPath(dir_inner_path + "/" + file_relative_path)
+                sha512sum = CryptHash.sha512sum(file_path)  # Calculate sha512 sum of file
+                if optional:
+                    self.log.info("- [OPTIONAL] %s (SHA512: %s)" % (file_relative_path, sha512sum))
+                    files_optional_node[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
+                else:
+                    self.log.info("- %s (SHA512: %s)" % (file_relative_path, sha512sum))
+                    files_node[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
+        return files_node, files_optional_node
+
     # Create and sign a content.json
     # Return: The new content if filewrite = False
     def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None):
@@ -253,35 +284,20 @@ class ContentManager(object):
                 content.update(extend)  # Add custom fields
 
         directory = self.toDir(self.site.storage.getPath(inner_path))
+        inner_directory = self.toDir(inner_path)
         self.log.info("Opening site data directory: %s..." % directory)
 
-        hashed_files = {}
         changed_files = [inner_path]
-        for root, dirs, files in os.walk(directory):
+        files_node, files_optional_node = self.hashFiles(self.toDir(inner_path), content.get("ignore"), content.get("optional"))
-            for file_name in files:
-                file_path = self.site.storage.getPath("%s/%s" % (root.strip("/"), file_name))
-                file_inner_path = re.sub(re.escape(directory), "", file_path)
 
-                if file_name == "content.json":
+        # Find changed files
-                    ignored = True
+        files_merged = files_node.copy()
-                elif content.get("ignore") and re.match(content["ignore"], file_inner_path):
+        files_merged.update(files_optional_node)
-                    ignored = True
+        for file_relative_path, file_details in files_merged.iteritems():
-                elif file_name.startswith("."):
+            old_hash = content["files"].get(file_relative_path, {}).get("sha512")
-                    ignored = True
+            new_hash = files_merged[file_relative_path]["sha512"]
-                else:
+            if old_hash != new_hash:
-                    ignored = False
+                changed_files.append(inner_directory + file_relative_path)
-
-                if ignored:  # Ignore content.json, definied regexp and files starting with .
-                    self.log.info("- [SKIPPED] %s" % file_inner_path)
-                else:
-                    sha512sum = CryptHash.sha512sum(file_path)  # Calculate sha512 sum of file
-                    self.log.info("- %s (SHA512: %s)" % (file_inner_path, sha512sum))
-                    hashed_files[file_inner_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
-                    if (
-                        file_inner_path in content["files"].keys()
-                        and hashed_files[file_inner_path]["sha512"] != content["files"][file_inner_path].get("sha512")
-                    ):
-                        changed_files.append(file_path)
 
         self.log.debug("Changed files: %s" % changed_files)
         if update_changed_files:
@@ -292,7 +308,9 @@ class ContentManager(object):
         self.log.info("Adding timestamp and sha512sums to new content.json...")
 
         new_content = content.copy()  # Create a copy of current content.json
-        new_content["files"] = hashed_files  # Add files sha512 hash
+        new_content["files"] = files_node  # Add files sha512 hash
+        if files_optional_node:
+            new_content["files_optional_node"] = files_optional_node
         new_content["modified"] = time.time()  # Add timestamp
         if inner_path == "content.json":
             new_content["address"] = self.site.address
@@ -336,7 +354,7 @@ class ContentManager(object):
             oldsign_content = json.dumps(new_content, sort_keys=True)
             new_content["sign"] = CryptBitcoin.signOld(oldsign_content, privatekey)
 
-        if not self.validContent(inner_path, new_content):
+        if not self.verifyContent(inner_path, new_content):
             self.log.error("Sign failed: Invalid content")
             return False
 
@@ -389,7 +407,7 @@ class ContentManager(object):
 
     # Checks if the content.json content is valid
     # Return: True or False
-    def validContent(self, inner_path, content):
+    def verifyContent(self, inner_path, content):
         content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()])  # Size of new content
         site_size = self.getTotalSize(ignore=inner_path) + content_size  # Site size without old content
         if site_size > self.site.settings.get("size", 0):
@@ -465,7 +483,7 @@ class ContentManager(object):
                     del(new_content["signs"])  # The file signed without the signs
                 sign_content = json.dumps(new_content, sort_keys=True)  # Dump the json to string to remove whitepsace
 
-                if not self.validContent(inner_path, new_content):
+                if not self.verifyContent(inner_path, new_content):
                     return False  # Content not valid (files too large, invalid files)
 
                 if signs:  # New style signing
@@ -527,6 +545,11 @@ class ContentManager(object):
             file_dir += "/"  # Add / at end if its not the root
         return file_dir
 
+    # Get dir from file
+    # Return: data/site/content.json -> data/site
+    def toFilename(self, inner_path):
+        return re.sub("^.*/", "", inner_path)
+
 
 if __name__ == "__main__":
     def testSign():

src/Site/SiteStorage.py

@@ -158,6 +158,19 @@ class SiteStorage:
         file_path = self.getPath(inner_path)
         os.unlink(file_path)
 
+    # List files from a directory
+    def list(self, dir_inner_path):
+        directory = self.getPath(dir_inner_path)
+        for root, dirs, files in os.walk(directory):
+            root = root.replace("\\", "/")
+            root_relative_path = re.sub("^%s" % re.escape(directory), "", root).lstrip("/")
+            for file_name in files:
+                if root_relative_path:  # Not root dir
+                    yield root_relative_path + "/" + file_name
+                else:
+                    yield file_name
+
+
     # Site content updated
     def onUpdated(self, inner_path):
         file_path = self.getPath(inner_path)
@@ -215,6 +228,9 @@ class SiteStorage:
         inner_path = inner_path.replace("\\", "/")  # Windows separator fix
         inner_path = re.sub("^%s/" % re.escape(self.directory), "", inner_path)  # Remove site directory if begins with it
         file_path = u"%s/%s" % (self.directory, inner_path)
+        if not inner_path:
+            return self.directory
+
         file_abspath = os.path.dirname(os.path.abspath(file_path))
         if ".." in file_path or not file_abspath.startswith(self.allowed_dir):
             raise Exception(u"File not allowed: %s" % file_path)
@@ -222,7 +238,10 @@ class SiteStorage:
 
     # Get site dir relative path
     def getInnerPath(self, path):
-        inner_path = re.sub("^%s/" % re.escape(self.directory), "", path)
+        if path == self.directory:
+            inner_path = ""
+        else:
+            inner_path = re.sub("^%s/" % re.escape(self.directory), "", path)
         return inner_path
 
     # Verify all files sha512sum using content.json

src/Test/TestPeer.py

@@ -22,7 +22,7 @@ class TestFileRequest:
         # Add file_server as peer to client
         peer_file_server = site_temp.addPeer("127.0.0.1", 1544)
 
-        assert peer_file_server.ping()
+        assert peer_file_server.ping() is not None
 
         assert peer_file_server in site_temp.peers.values()
         peer_file_server.remove()

src/Test/TestSite.py

@@ -62,3 +62,36 @@ class TestSite:
         assert new_site.address in SiteManager.site_manager.sites
         SiteManager.site_manager.delete(new_site.address)
         assert new_site.address not in SiteManager.site_manager.sites
+
+    @pytest.mark.parametrize("inner_path", ["content.json", "data/test_include/content.json", "data/users/content.json"])
+    def testSign(self, site, inner_path):
+        # Bad privatekey
+        assert not site.content_manager.sign(inner_path, privatekey="5aaa3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMnaa", filewrite=False)
+
+        # Good privatekey
+        content = site.content_manager.sign(inner_path, privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
+        content_old = site.content_manager.contents[inner_path]  # Content before the sign
+        assert not content_old == content  # Timestamp changed
+        assert site.address in content["signs"]  # Used the site's private key to sign
+        if inner_path == "content.json":
+            assert len(content["files"]) == 24
+        elif inner_path == "data/test-include/content.json":
+            assert len(content["files"]) == 1
+        elif inner_path == "data/users/content.json":
+            assert len(content["files"]) == 0
+
+        # Everything should be same as before except the modified timestamp and the signs
+        assert (
+            {key: val for key, val in content_old.items() if key not in ["modified", "signs", "sign"]}
+            ==
+            {key: val for key, val in content.items() if key not in ["modified", "signs", "sign"]}
+        )
+
+    def testSignOptionalFiles(self, site):
+        site.content_manager.contents["content.json"]["optional"] = "((data/img/zero.*))"
+        content_optional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
+
+        del site.content_manager.contents["content.json"]["optional"]
+        content_nooptional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
+
+        assert len(content_nooptional["files"]) > len(content_optional["files"])

src/Test/TestSiteStorage.py

@@ -0,0 +1,15 @@
+import shutil
+import os
+
+import pytest
+from Site import SiteManager
+
+
+@pytest.mark.usefixtures("resetSettings")
+class TestSiteStorage:
+    def testList(self, site):
+    	list_root = list(site.storage.list(""))
+    	assert "content.json" in list_root
+    	assert "css/all.css" in list_root
+
+        assert list(site.storage.list("data-default")) == ["data.json", "users/content-default.json"]

src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json

@@ -1,133 +1,133 @@
 {
- "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", 
+ "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT",
- "background-color": "white", 
+ "background-color": "white",
- "description": "Blogging platform Demo", 
+ "description": "Blogging platform Demo",
- "domain": "Blog.ZeroNetwork.bit", 
+ "domain": "Blog.ZeroNetwork.bit",
  "files": {
   "css/all.css": {
-   "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", 
+   "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc",
    "size": 112710
-  }, 
+  },
   "data-default/data.json": {
-   "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", 
+   "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2",
    "size": 196
-  }, 
+  },
   "data-default/users/content-default.json": {
-   "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", 
+   "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa",
    "size": 679
-  }, 
+  },
   "data/data.json": {
-   "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", 
+   "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52",
    "size": 31126
-  }, 
+  },
   "data/img/autoupdate.png": {
-   "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", 
+   "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71",
    "size": 24460
-  }, 
+  },
   "data/img/direct_domains.png": {
-   "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", 
+   "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719",
    "size": 16185
-  }, 
+  },
   "data/img/domain.png": {
-   "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", 
+   "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a",
    "size": 11881
-  }, 
+  },
   "data/img/memory.png": {
-   "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", 
+   "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e",
    "size": 12775
-  }, 
+  },
   "data/img/multiuser.png": {
-   "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", 
+   "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6",
    "size": 29480
-  }, 
+  },
   "data/img/progressbar.png": {
-   "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", 
+   "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf",
    "size": 13294
-  }, 
+  },
   "data/img/slides.png": {
-   "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", 
+   "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e",
    "size": 14439
-  }, 
+  },
   "data/img/slots_memory.png": {
-   "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", 
+   "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29",
    "size": 9488
-  }, 
+  },
   "data/img/trayicon.png": {
-   "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", 
+   "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f",
    "size": 19040
-  }, 
+  },
   "data/img/zeroblog-comments.png": {
-   "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", 
+   "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4",
    "size": 24001
-  }, 
+  },
   "data/img/zeroid.png": {
-   "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", 
+   "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b",
    "size": 18875
-  }, 
+  },
   "data/img/zeroname.png": {
-   "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", 
+   "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d",
    "size": 36031
-  }, 
+  },
   "data/img/zerotalk-mark.png": {
-   "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", 
+   "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af",
    "size": 44862
-  }, 
+  },
   "data/img/zerotalk-upvote.png": {
-   "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", 
+   "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732",
    "size": 41092
-  }, 
+  },
   "data/img/zerotalk.png": {
-   "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", 
+   "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8",
    "size": 26606
-  }, 
+  },
   "data/test_include/data.json": {
-   "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", 
+   "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906",
    "size": 505
-  }, 
+  },
   "dbschema.json": {
-   "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", 
+   "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8",
    "size": 1529
-  }, 
+  },
   "img/loading.gif": {
-   "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", 
+   "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00",
    "size": 723
-  }, 
+  },
   "index.html": {
-   "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", 
+   "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3",
    "size": 4667
-  }, 
+  },
   "js/all.js": {
-   "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", 
+   "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c",
    "size": 201178
   }
- }, 
+ },
- "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)", 
+ "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)",
  "includes": {
   "data/test_include/content.json": {
-   "added": 1424976057, 
+   "added": 1424976057,
-   "files_allowed": "data.json", 
+   "files_allowed": "data.json",
-   "includes_allowed": false, 
+   "includes_allowed": false,
-   "max_size": 20000, 
+   "max_size": 20000,
-   "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], 
+   "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ],
-   "signers_required": 1, 
+   "signers_required": 1,
-   "user_id": 47, 
+   "user_id": 47,
    "user_name": "test"
-  }, 
+  },
   "data/users/content.json": {
-   "signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ], 
+   "signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ],
    "signers_required": 1
   }
- }, 
+ },
- "modified": 1443088239.123, 
+ "modified": 1443393859.801,
  "sign": [
-  37796247323133993908968541760020085519225012317332056166386012116450888757672, 
+  30041653970398729892154852118727733790145614202537425646336077462070348808967,
-  8182016604193300184892407269063757269964429504791487428802219119125679030316
+  96823925597554846684463773054016176426938620086211253074026312396122955360853
- ], 
+ ],
- "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", 
+ "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=",
  "signs": {
-  "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HJ+SuvmYh1DIyvqypUobaspZ3heUfYWoN34S4c2la5NgcBmpZ/YN4Xzi6wtP20W8DePXdsYMC0Azr+L8ZF7FAk4="
+  "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HKBBvaOi1v20ZuSVORtD4bBRRgf/85QDVy4HaaPX3fFDAKYmvUWK+Jbp3yIGElMmPoO2+YljFLyromAoEwWd6Eg="
- }, 
+ },
- "signs_required": 1, 
+ "signs_required": 1,
- "title": "ZeroBlog", 
+ "title": "ZeroBlog",
  "zeronet_version": "0.3.2"
 }