From a7d8d488da71804f42e88807eb6c761a767c771e Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Mon, 28 Sep 2015 22:07:26 +0200 Subject: [PATCH] Rev465, Display memory dump only in debug mode, Use sys.exit in trayicon, Optional files hashing, List function in SiteStorage, Test signing content, Test sign optional files, Test site storage --- plugins/Stats/StatsPlugin.py | 19 +++ plugins/Trayicon/TrayiconPlugin.py | 6 +- src/Config.py | 2 +- src/Content/ContentManager.py | 81 ++++++---- src/Site/SiteStorage.py | 21 ++- src/Test/TestPeer.py | 2 +- src/Test/TestSite.py | 33 ++++ src/Test/TestSiteStorage.py | 15 ++ .../content.json | 144 +++++++++--------- 9 files changed, 216 insertions(+), 107 deletions(-) create mode 100644 src/Test/TestSiteStorage.py diff --git a/plugins/Stats/StatsPlugin.py b/plugins/Stats/StatsPlugin.py index 09ba2ee4..01b3d249 100644 --- a/plugins/Stats/StatsPlugin.py +++ b/plugins/Stats/StatsPlugin.py @@ -147,8 +147,13 @@ class UiRequestPlugin(object): yield "
" yield "" + # No more if not in debug mode + if not config.debug: + raise StopIteration + # Object types + obj_count = {} for obj in gc.get_objects(): obj_type = str(type(obj)) @@ -250,10 +255,17 @@ class UiRequestPlugin(object): yield "Done in %.1f" % (time.time() - s) def actionDumpobj(self): + import gc import sys self.sendHeader() + + # No more if not in debug mode + if not config.debug: + yield "Not in debug mode" + raise StopIteration + class_filter = self.get.get("class") yield """ @@ -276,10 +288,17 @@ class UiRequestPlugin(object): gc.collect() # Implicit grabage collection def actionListobj(self): + import gc import sys self.sendHeader() + + # No more if not in debug mode + if not config.debug: + yield "Not in debug mode" + raise StopIteration + type_filter = self.get.get("type") yield """ diff --git a/plugins/Trayicon/TrayiconPlugin.py b/plugins/Trayicon/TrayiconPlugin.py index 6579612c..e962281d 100644 --- a/plugins/Trayicon/TrayiconPlugin.py +++ b/plugins/Trayicon/TrayiconPlugin.py @@ -63,9 +63,9 @@ class ActionsPlugin(object): def quit(self): self.icon.die() time.sleep(0.1) - self.main.ui_server.stop() - self.main.file_server.stop() - # sys.exit() + sys.exit() + #self.main.ui_server.stop() + #self.main.file_server.stop() def opensite(self, url): import webbrowser diff --git a/src/Config.py b/src/Config.py index 5ddfcd92..62c76faa 100644 --- a/src/Config.py +++ b/src/Config.py @@ -8,7 +8,7 @@ class Config(object): def __init__(self, argv): self.version = "0.3.2" - self.rev = 452 + self.rev = 465 self.argv = argv self.action = None self.createParser() diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index fa0ae8f1..3fee5de4 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -237,6 +237,37 @@ class ContentManager(object): return rules + # Hash files in directory + def hashFiles(self, dir_inner_path, ignore_pattern=None, optional_pattern=None): + files_node = {} + files_optional_node = {} + + for file_relative_path in self.site.storage.list(dir_inner_path): + file_name = self.toFilename(file_relative_path) + + ignored = optional = False + if file_name == "content.json": + ignored = True + elif ignore_pattern and re.match(ignore_pattern, file_relative_path): + ignored = True + elif file_name.startswith("."): + ignored = True + elif optional_pattern and re.match(optional_pattern, file_relative_path): + optional = True + + if ignored: # Ignore content.json, definied regexp and files starting with . + self.log.info("- [SKIPPED] %s" % file_relative_path) + else: + file_path = self.site.storage.getPath(dir_inner_path + "/" + file_relative_path) + sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file + if optional: + self.log.info("- [OPTIONAL] %s (SHA512: %s)" % (file_relative_path, sha512sum)) + files_optional_node[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)} + else: + self.log.info("- %s (SHA512: %s)" % (file_relative_path, sha512sum)) + files_node[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)} + return files_node, files_optional_node + # Create and sign a content.json # Return: The new content if filewrite = False def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None): @@ -253,35 +284,20 @@ class ContentManager(object): content.update(extend) # Add custom fields directory = self.toDir(self.site.storage.getPath(inner_path)) + inner_directory = self.toDir(inner_path) self.log.info("Opening site data directory: %s..." % directory) - hashed_files = {} changed_files = [inner_path] - for root, dirs, files in os.walk(directory): - for file_name in files: - file_path = self.site.storage.getPath("%s/%s" % (root.strip("/"), file_name)) - file_inner_path = re.sub(re.escape(directory), "", file_path) + files_node, files_optional_node = self.hashFiles(self.toDir(inner_path), content.get("ignore"), content.get("optional")) - if file_name == "content.json": - ignored = True - elif content.get("ignore") and re.match(content["ignore"], file_inner_path): - ignored = True - elif file_name.startswith("."): - ignored = True - else: - ignored = False - - if ignored: # Ignore content.json, definied regexp and files starting with . - self.log.info("- [SKIPPED] %s" % file_inner_path) - else: - sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file - self.log.info("- %s (SHA512: %s)" % (file_inner_path, sha512sum)) - hashed_files[file_inner_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)} - if ( - file_inner_path in content["files"].keys() - and hashed_files[file_inner_path]["sha512"] != content["files"][file_inner_path].get("sha512") - ): - changed_files.append(file_path) + # Find changed files + files_merged = files_node.copy() + files_merged.update(files_optional_node) + for file_relative_path, file_details in files_merged.iteritems(): + old_hash = content["files"].get(file_relative_path, {}).get("sha512") + new_hash = files_merged[file_relative_path]["sha512"] + if old_hash != new_hash: + changed_files.append(inner_directory + file_relative_path) self.log.debug("Changed files: %s" % changed_files) if update_changed_files: @@ -292,7 +308,9 @@ class ContentManager(object): self.log.info("Adding timestamp and sha512sums to new content.json...") new_content = content.copy() # Create a copy of current content.json - new_content["files"] = hashed_files # Add files sha512 hash + new_content["files"] = files_node # Add files sha512 hash + if files_optional_node: + new_content["files_optional_node"] = files_optional_node new_content["modified"] = time.time() # Add timestamp if inner_path == "content.json": new_content["address"] = self.site.address @@ -336,7 +354,7 @@ class ContentManager(object): oldsign_content = json.dumps(new_content, sort_keys=True) new_content["sign"] = CryptBitcoin.signOld(oldsign_content, privatekey) - if not self.validContent(inner_path, new_content): + if not self.verifyContent(inner_path, new_content): self.log.error("Sign failed: Invalid content") return False @@ -389,7 +407,7 @@ class ContentManager(object): # Checks if the content.json content is valid # Return: True or False - def validContent(self, inner_path, content): + def verifyContent(self, inner_path, content): content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content site_size = self.getTotalSize(ignore=inner_path) + content_size # Site size without old content if site_size > self.site.settings.get("size", 0): @@ -465,7 +483,7 @@ class ContentManager(object): del(new_content["signs"]) # The file signed without the signs sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace - if not self.validContent(inner_path, new_content): + if not self.verifyContent(inner_path, new_content): return False # Content not valid (files too large, invalid files) if signs: # New style signing @@ -527,6 +545,11 @@ class ContentManager(object): file_dir += "/" # Add / at end if its not the root return file_dir + # Get dir from file + # Return: data/site/content.json -> data/site + def toFilename(self, inner_path): + return re.sub("^.*/", "", inner_path) + if __name__ == "__main__": def testSign(): diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index 9368fbb6..7856fad8 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -158,6 +158,19 @@ class SiteStorage: file_path = self.getPath(inner_path) os.unlink(file_path) + # List files from a directory + def list(self, dir_inner_path): + directory = self.getPath(dir_inner_path) + for root, dirs, files in os.walk(directory): + root = root.replace("\\", "/") + root_relative_path = re.sub("^%s" % re.escape(directory), "", root).lstrip("/") + for file_name in files: + if root_relative_path: # Not root dir + yield root_relative_path + "/" + file_name + else: + yield file_name + + # Site content updated def onUpdated(self, inner_path): file_path = self.getPath(inner_path) @@ -215,6 +228,9 @@ class SiteStorage: inner_path = inner_path.replace("\\", "/") # Windows separator fix inner_path = re.sub("^%s/" % re.escape(self.directory), "", inner_path) # Remove site directory if begins with it file_path = u"%s/%s" % (self.directory, inner_path) + if not inner_path: + return self.directory + file_abspath = os.path.dirname(os.path.abspath(file_path)) if ".." in file_path or not file_abspath.startswith(self.allowed_dir): raise Exception(u"File not allowed: %s" % file_path) @@ -222,7 +238,10 @@ class SiteStorage: # Get site dir relative path def getInnerPath(self, path): - inner_path = re.sub("^%s/" % re.escape(self.directory), "", path) + if path == self.directory: + inner_path = "" + else: + inner_path = re.sub("^%s/" % re.escape(self.directory), "", path) return inner_path # Verify all files sha512sum using content.json diff --git a/src/Test/TestPeer.py b/src/Test/TestPeer.py index ae5a6b84..5c9a7aba 100644 --- a/src/Test/TestPeer.py +++ b/src/Test/TestPeer.py @@ -22,7 +22,7 @@ class TestFileRequest: # Add file_server as peer to client peer_file_server = site_temp.addPeer("127.0.0.1", 1544) - assert peer_file_server.ping() + assert peer_file_server.ping() is not None assert peer_file_server in site_temp.peers.values() peer_file_server.remove() diff --git a/src/Test/TestSite.py b/src/Test/TestSite.py index 5aa14069..3aaad8e5 100644 --- a/src/Test/TestSite.py +++ b/src/Test/TestSite.py @@ -62,3 +62,36 @@ class TestSite: assert new_site.address in SiteManager.site_manager.sites SiteManager.site_manager.delete(new_site.address) assert new_site.address not in SiteManager.site_manager.sites + + @pytest.mark.parametrize("inner_path", ["content.json", "data/test_include/content.json", "data/users/content.json"]) + def testSign(self, site, inner_path): + # Bad privatekey + assert not site.content_manager.sign(inner_path, privatekey="5aaa3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMnaa", filewrite=False) + + # Good privatekey + content = site.content_manager.sign(inner_path, privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) + content_old = site.content_manager.contents[inner_path] # Content before the sign + assert not content_old == content # Timestamp changed + assert site.address in content["signs"] # Used the site's private key to sign + if inner_path == "content.json": + assert len(content["files"]) == 24 + elif inner_path == "data/test-include/content.json": + assert len(content["files"]) == 1 + elif inner_path == "data/users/content.json": + assert len(content["files"]) == 0 + + # Everything should be same as before except the modified timestamp and the signs + assert ( + {key: val for key, val in content_old.items() if key not in ["modified", "signs", "sign"]} + == + {key: val for key, val in content.items() if key not in ["modified", "signs", "sign"]} + ) + + def testSignOptionalFiles(self, site): + site.content_manager.contents["content.json"]["optional"] = "((data/img/zero.*))" + content_optional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) + + del site.content_manager.contents["content.json"]["optional"] + content_nooptional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) + + assert len(content_nooptional["files"]) > len(content_optional["files"]) diff --git a/src/Test/TestSiteStorage.py b/src/Test/TestSiteStorage.py new file mode 100644 index 00000000..91eb43c4 --- /dev/null +++ b/src/Test/TestSiteStorage.py @@ -0,0 +1,15 @@ +import shutil +import os + +import pytest +from Site import SiteManager + + +@pytest.mark.usefixtures("resetSettings") +class TestSiteStorage: + def testList(self, site): + list_root = list(site.storage.list("")) + assert "content.json" in list_root + assert "css/all.css" in list_root + + assert list(site.storage.list("data-default")) == ["data.json", "users/content-default.json"] diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json index 3474221b..fdc49a19 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json @@ -1,133 +1,133 @@ { - "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", - "background-color": "white", - "description": "Blogging platform Demo", - "domain": "Blog.ZeroNetwork.bit", + "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", + "background-color": "white", + "description": "Blogging platform Demo", + "domain": "Blog.ZeroNetwork.bit", "files": { "css/all.css": { - "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", + "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", "size": 112710 - }, + }, "data-default/data.json": { - "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", + "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", "size": 196 - }, + }, "data-default/users/content-default.json": { - "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", + "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", "size": 679 - }, + }, "data/data.json": { - "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", + "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", "size": 31126 - }, + }, "data/img/autoupdate.png": { - "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", + "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", "size": 24460 - }, + }, "data/img/direct_domains.png": { - "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", + "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", "size": 16185 - }, + }, "data/img/domain.png": { - "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", + "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", "size": 11881 - }, + }, "data/img/memory.png": { - "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", + "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", "size": 12775 - }, + }, "data/img/multiuser.png": { - "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", + "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", "size": 29480 - }, + }, "data/img/progressbar.png": { - "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", + "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", "size": 13294 - }, + }, "data/img/slides.png": { - "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", + "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", "size": 14439 - }, + }, "data/img/slots_memory.png": { - "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", + "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", "size": 9488 - }, + }, "data/img/trayicon.png": { - "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", + "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", "size": 19040 - }, + }, "data/img/zeroblog-comments.png": { - "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", + "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", "size": 24001 - }, + }, "data/img/zeroid.png": { - "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", + "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", "size": 18875 - }, + }, "data/img/zeroname.png": { - "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", + "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", "size": 36031 - }, + }, "data/img/zerotalk-mark.png": { - "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", + "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", "size": 44862 - }, + }, "data/img/zerotalk-upvote.png": { - "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", + "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", "size": 41092 - }, + }, "data/img/zerotalk.png": { - "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", + "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", "size": 26606 - }, + }, "data/test_include/data.json": { - "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", + "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", "size": 505 - }, + }, "dbschema.json": { - "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", + "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", "size": 1529 - }, + }, "img/loading.gif": { - "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", + "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", "size": 723 - }, + }, "index.html": { - "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", + "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", "size": 4667 - }, + }, "js/all.js": { - "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", + "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", "size": 201178 } - }, - "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)", + }, + "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)", "includes": { "data/test_include/content.json": { - "added": 1424976057, - "files_allowed": "data.json", - "includes_allowed": false, - "max_size": 20000, - "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], - "signers_required": 1, - "user_id": 47, + "added": 1424976057, + "files_allowed": "data.json", + "includes_allowed": false, + "max_size": 20000, + "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], + "signers_required": 1, + "user_id": 47, "user_name": "test" - }, + }, "data/users/content.json": { - "signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ], + "signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ], "signers_required": 1 } - }, - "modified": 1443088239.123, + }, + "modified": 1443393859.801, "sign": [ - 37796247323133993908968541760020085519225012317332056166386012116450888757672, - 8182016604193300184892407269063757269964429504791487428802219119125679030316 - ], - "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", + 30041653970398729892154852118727733790145614202537425646336077462070348808967, + 96823925597554846684463773054016176426938620086211253074026312396122955360853 + ], + "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", "signs": { - "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HJ+SuvmYh1DIyvqypUobaspZ3heUfYWoN34S4c2la5NgcBmpZ/YN4Xzi6wtP20W8DePXdsYMC0Azr+L8ZF7FAk4=" - }, - "signs_required": 1, - "title": "ZeroBlog", + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HKBBvaOi1v20ZuSVORtD4bBRRgf/85QDVy4HaaPX3fFDAKYmvUWK+Jbp3yIGElMmPoO2+YljFLyromAoEwWd6Eg=" + }, + "signs_required": 1, + "title": "ZeroBlog", "zeronet_version": "0.3.2" } \ No newline at end of file