From 9d7d4f1552c1d15aee680e52d8b6c6f687afc635 Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Thu, 1 Oct 2015 01:35:13 +0200 Subject: [PATCH] Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers --- requirements.txt | 4 +- src/Config.py | 2 +- src/Content/ContentManager.py | 77 +++++--- src/Peer/Peer.py | 38 +++- src/Site/Site.py | 4 +- src/Site/SiteStorage.py | 16 +- src/Test/Spy.py | 17 ++ src/Test/TestContent.py | 64 ++++++- src/Test/TestContentUser.py | 60 ++++++ src/Test/TestHelper.py | 52 ++++-- src/Test/TestSite.py | 33 ---- src/Test/TestSiteDownload.py | 63 +++++++ src/Test/TestSiteStorage.py | 12 +- src/Test/conftest.py | 3 +- .../content.json | 175 +++++++++--------- .../data/optional.txt | 1 + .../content.json | 31 ++-- .../peanut-butter-jelly-time.gif | Bin 0 -> 1606 bytes .../content.json | 24 +-- .../data/users/content.json | 6 +- src/Worker/WorkerManager.py | 4 +- src/util/helper.py | 20 +- 22 files changed, 486 insertions(+), 220 deletions(-) create mode 100644 src/Test/Spy.py create mode 100644 src/Test/TestSiteDownload.py create mode 100644 src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/optional.txt create mode 100644 src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif diff --git a/requirements.txt b/requirements.txt index 61555a22..f60dc30b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -gevent==1.0.1 -msgpack-python==0.4.4 +gevent>=1.0.1 +msgpack-python>=0.4.4 diff --git a/src/Config.py b/src/Config.py index 62c76faa..6bd44974 100644 --- a/src/Config.py +++ b/src/Config.py @@ -8,7 +8,7 @@ class Config(object): def __init__(self, argv): self.version = "0.3.2" - self.rev = 465 + self.rev = 467 self.argv = argv self.action = None self.createParser() diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index 3fee5de4..094a11b2 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -9,7 +9,7 @@ import gevent from Debug import Debug from Crypt import CryptHash from Config import config - +from util import helper class ContentManager(object): @@ -26,8 +26,8 @@ class ContentManager(object): content_inner_path = content_inner_path.strip("/") # Remove / from begning old_content = self.contents.get(content_inner_path) content_path = self.site.storage.getPath(content_inner_path) - content_dir = self.toDir(self.site.storage.getPath(content_inner_path)) - content_inner_dir = self.toDir(content_inner_path) + content_dir = helper.getDirname(self.site.storage.getPath(content_inner_path)) + content_inner_dir = helper.getDirname(content_inner_path) if os.path.isfile(content_path): try: @@ -140,16 +140,29 @@ class ContentManager(object): while True: content_inner_path = "%s/content.json" % "/".join(dirs) content = self.contents.get(content_inner_path.strip("/")) - if content and "files" in content: # Check if content.json exists + + # Check in files + if content and "files" in content: back = content["files"].get("/".join(inner_path_parts)) if back: back["content_inner_path"] = content_inner_path + back["optional"] = False return back - if content and "user_contents" in content: # User dir + # Check in optional files + if content and "files_optional" in content: # Check if file in this content.json + back = content["files_optional"].get("/".join(inner_path_parts)) + if back: + back["content_inner_path"] = content_inner_path + back["optional"] = True + return back + + # Return the rules if user dir + if content and "user_contents" in content: back = content["user_contents"] # Content.json is in the users dir back["content_inner_path"] = re.sub("(.*)/.*?$", "\\1/content.json", inner_path) + back["optional"] = None return back # No inner path in this dir, lets try the parent dir @@ -234,6 +247,7 @@ class ContentManager(object): rules["signers"] = [] rules["signers"].append(user_address) # Add user as valid signer rules["user_address"] = user_address + rules["includes_allowed"] = False return rules @@ -243,7 +257,7 @@ class ContentManager(object): files_optional_node = {} for file_relative_path in self.site.storage.list(dir_inner_path): - file_name = self.toFilename(file_relative_path) + file_name = helper.getFilename(file_relative_path) ignored = optional = False if file_name == "content.json": @@ -283,12 +297,12 @@ class ContentManager(object): if extend: content.update(extend) # Add custom fields - directory = self.toDir(self.site.storage.getPath(inner_path)) - inner_directory = self.toDir(inner_path) + directory = helper.getDirname(self.site.storage.getPath(inner_path)) + inner_directory = helper.getDirname(inner_path) self.log.info("Opening site data directory: %s..." % directory) changed_files = [inner_path] - files_node, files_optional_node = self.hashFiles(self.toDir(inner_path), content.get("ignore"), content.get("optional")) + files_node, files_optional_node = self.hashFiles(helper.getDirname(inner_path), content.get("ignore"), content.get("optional")) # Find changed files files_merged = files_node.copy() @@ -310,13 +324,17 @@ class ContentManager(object): new_content = content.copy() # Create a copy of current content.json new_content["files"] = files_node # Add files sha512 hash if files_optional_node: - new_content["files_optional_node"] = files_optional_node + new_content["files_optional"] = files_optional_node + elif "files_optional" in new_content: + del new_content["files_optional"] + new_content["modified"] = time.time() # Add timestamp if inner_path == "content.json": new_content["address"] = self.site.address new_content["zeronet_version"] = config.version new_content["signs_required"] = content.get("signs_required", 1) + # Verify private key from Crypt import CryptBitcoin self.log.info("Verifying private key...") privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey) @@ -409,6 +427,7 @@ class ContentManager(object): # Return: True or False def verifyContent(self, inner_path, content): content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content + content_size_optional = sum([file["size"] for file in content.get("files_optional", {}).values()]) site_size = self.getTotalSize(ignore=inner_path) + content_size # Site size without old content if site_size > self.site.settings.get("size", 0): self.site.settings["size"] = site_size # Save to settings if larger @@ -433,23 +452,34 @@ class ContentManager(object): return False # Check include size limit - if rules.get("max_size"): # Include size limit + if rules.get("max_size") is not None: # Include size limit if content_size > rules["max_size"]: self.log.error("%s: Include too large %s > %s" % (inner_path, content_size, rules["max_size"])) return False - # Check if content includes allowed - if rules.get("includes_allowed") is False and content.get("includes"): - self.log.error("%s: Includes not allowed" % inner_path) - return False # Includes not allowed + if rules.get("max_size_optional") is not None: # Include optional files limit + if content_size_optional > rules["max_size_optional"]: + self.log.error("%s: Include optional files too large %s > %s" % (inner_path, content_size_optional, rules["max_size_optional"])) + return False # Filename limit if rules.get("files_allowed"): for file_inner_path in content["files"].keys(): if not re.match("^%s$" % rules["files_allowed"], file_inner_path): - self.log.error("%s: File not allowed" % file_inner_path) + self.log.error("%s %s: File not allowed" % (inner_path, file_inner_path)) return False + if rules.get("files_allowed_optional"): + for file_inner_path in content.get("files_optional", {}).keys(): + if not re.match("^%s$" % rules["files_allowed_optional"], file_inner_path): + self.log.error("%s %s: Optional file not allowed" % (inner_path, file_inner_path)) + return False + + # Check if content includes allowed + if rules.get("includes_allowed") is False and content.get("includes"): + self.log.error("%s: Includes not allowed" % inner_path) + return False # Includes not allowed + return True # All good # Verify file validity @@ -507,7 +537,7 @@ class ContentManager(object): valid_signs += CryptBitcoin.verify(sign_content, address, signs[address]) if valid_signs >= signs_required: break # Break if we has enough signs - + self.log.debug("%s: Valid signs: %s/%s" % (inner_path, valid_signs, signs_required)) return valid_signs >= signs_required else: # Old style signing return CryptBitcoin.verify(sign_content, self.site.address, sign) @@ -537,19 +567,6 @@ class ContentManager(object): self.log.error("File not in content.json: %s" % inner_path) return False - # Get dir from file - # Return: data/site/content.json -> data/site - def toDir(self, inner_path): - file_dir = re.sub("[^/]*?$", "", inner_path).strip("/") - if file_dir: - file_dir += "/" # Add / at end if its not the root - return file_dir - - # Get dir from file - # Return: data/site/content.json -> data/site - def toFilename(self, inner_path): - return re.sub("^.*/", "", inner_path) - if __name__ == "__main__": def testSign(): diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 77b689b7..af3ca228 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -1,6 +1,8 @@ import logging -import gevent import time +import array + +import gevent from cStringIO import StringIO from Debug import Debug @@ -14,8 +16,8 @@ if config.use_tempfiles: # Communicate remote peers class Peer(object): __slots__ = ( - "ip", "port", "site", "key", "connection", "last_found", "last_response", - "last_ping", "added", "connection_error", "hash_failed", "download_bytes", "download_time" + "ip", "port", "site", "key", "connection", "last_found", "last_response", "last_ping", "last_hashfield", + "hashfield", "added", "connection_error", "hash_failed", "download_bytes", "download_time" ) def __init__(self, ip, port, site=None): @@ -25,6 +27,8 @@ class Peer(object): self.key = "%s:%s" % (ip, port) self.connection = None + self.hashfield = array.array("H") # Got optional files hash_id + self.last_hashfield = None # Last time hashfiled downloaded self.last_found = time.time() # Time of last found in the torrent tracker self.last_response = None # Time of last successful response from peer self.last_ping = None # Last response time for ping @@ -230,6 +234,34 @@ class Peer(object): if self.connection: self.connection.close() + # - HASHFIELD - + + def updateHashfield(self, force=False): + # Don't update hashfield again in 15 min + if self.last_hashfield and time.time() - self.last_hashfield > 60 * 15 and not force: + return False + + response = self.request("getHashfield", {"site": self.site.address}) + if not response or "error" in response: + return False + self.last_hashfield = time.time() + self.hashfield = response["hashfield"] + + return self.hashfield + + def setHashfield(self, hashfield_dump): + self.hashfield.fromstring(hashfield_dump) + + def hasHash(self, hash_id): + return hash_id in self.hashfield + + # Return: ["ip:port", "ip:port",...] + def findHash(self, hash_id): + response = self.request("findHash", {"site": self.site.address, "hash_id": hash_id}) + if not response or "error" in response: + return False + return [helper.unpackAddress(peer) for peer in response["peers"]] + # - EVENTS - # On connection error diff --git a/src/Site/Site.py b/src/Site/Site.py index 3a6f84a0..cdaad9d8 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -112,7 +112,7 @@ class Site: s = time.time() self.log.debug("Downloading %s..." % inner_path) found = self.needFile(inner_path, update=self.bad_files.get(inner_path)) - content_inner_dir = self.content_manager.toDir(inner_path) + content_inner_dir = helper.getDirname(inner_path) if not found: self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications)) if check_modifications: # Download failed, but check modifications if its succed later @@ -386,7 +386,7 @@ class Site: # Copy files for content_inner_path, content in self.content_manager.contents.items(): for file_relative_path in sorted(content["files"].keys()): - file_inner_path = self.content_manager.toDir(content_inner_path) + file_relative_path # Relative to content.json + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to content.json file_inner_path = file_inner_path.strip("/") # Strip leading / if file_inner_path.split("/")[0] in default_dirs: # Dont copy directories that has -default postfixed alternative self.log.debug("[SKIP] %s (has default alternative)" % file_inner_path) diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index 7856fad8..5b87f306 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -3,7 +3,6 @@ import re import shutil import json import time -import sys import sqlite3 import gevent.event @@ -11,6 +10,7 @@ import gevent.event from Db import Db from Debug import Debug from Config import config +from util import helper class SiteStorage: @@ -98,7 +98,7 @@ class SiteStorage: for file_relative_path in content["files"].keys(): if not file_relative_path.endswith(".json"): continue # We only interesed in json files - content_inner_path_dir = self.site.content_manager.toDir(content_inner_path) # Content.json dir relative to site + content_inner_path_dir = helper.getDirname(content_inner_path) # Content.json dir relative to site file_inner_path = content_inner_path_dir + file_relative_path # File Relative to site dir file_inner_path = file_inner_path.strip("/") # Strip leading / file_path = self.getPath(file_inner_path) @@ -170,7 +170,6 @@ class SiteStorage: else: yield file_name - # Site content updated def onUpdated(self, inner_path): file_path = self.getPath(inner_path) @@ -255,7 +254,7 @@ class SiteStorage: self.log.debug("[MISSING] %s" % content_inner_path) bad_files.append(content_inner_path) for file_relative_path in content["files"].keys(): - file_inner_path = self.site.content_manager.toDir(content_inner_path) + file_relative_path # Relative to site dir + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir file_inner_path = file_inner_path.strip("/") # Strip leading / file_path = self.getPath(file_inner_path) if not os.path.isfile(file_path): @@ -304,8 +303,13 @@ class SiteStorage: files = [] # Get filenames for content_inner_path, content in self.site.content_manager.contents.items(): files.append(content_inner_path) - for file_relative_path in content["files"].keys(): - file_inner_path = self.site.content_manager.toDir(content_inner_path) + file_relative_path # Relative to site dir + # Add normal files + for file_relative_path in content.get("files", {}).keys(): + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir + files.append(file_inner_path) + # Add optional files + for file_relative_path in content.get("files_optional", {}).keys(): + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir files.append(file_inner_path) for inner_path in files: diff --git a/src/Test/Spy.py b/src/Test/Spy.py new file mode 100644 index 00000000..c017dea9 --- /dev/null +++ b/src/Test/Spy.py @@ -0,0 +1,17 @@ +class Spy: + def __init__(self, obj, func_name): + self.obj = obj + self.func_name = func_name + self.func_original = getattr(self.obj, func_name) + self.calls = [] + + def __enter__(self, *args, **kwargs): + def loggedFunc(cls, *args, **kwags): + print "Logging", self, args, kwargs + self.calls.append(args) + return self.func_original(cls, *args, **kwargs) + setattr(self.obj, self.func_name, loggedFunc) + return self.calls + + def __exit__(self, *args, **kwargs): + setattr(self.obj, self.func_name, self.func_original) \ No newline at end of file diff --git a/src/Test/TestContent.py b/src/Test/TestContent.py index c7130a5b..5303287e 100644 --- a/src/Test/TestContent.py +++ b/src/Test/TestContent.py @@ -9,7 +9,7 @@ from Crypt import CryptBitcoin @pytest.mark.usefixtures("resetSettings") class TestContent: - def testIncludes(self, site): + def testInclude(self, site): # Rules defined in parent content.json rules = site.content_manager.getRules("data/test_include/content.json") @@ -34,7 +34,7 @@ class TestContent: # Valid signers for root content.json assert site.content_manager.getValidSigners("content.json") == ["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"] - def testLimits(self, site): + def testInlcudeLimits(self, site): privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # Data validation data_dict = { @@ -48,7 +48,7 @@ class TestContent: } # Normal data - data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) # Reset @@ -56,7 +56,7 @@ class TestContent: # Too large data_dict["files"]["data.json"]["size"] = 200000 # Emulate 2MB sized data.json - data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) assert not site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) # Reset @@ -65,7 +65,7 @@ class TestContent: # Not allowed file data_dict["files"]["notallowed.exe"] = data_dict["files"]["data.json"] - data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) assert not site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) # Reset @@ -73,6 +73,58 @@ class TestContent: del data_dict["signs"] # Should work again - data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) + + @pytest.mark.parametrize("inner_path", ["content.json", "data/test_include/content.json", "data/users/content.json"]) + def testSign(self, site, inner_path): + # Bad privatekey + assert not site.content_manager.sign(inner_path, privatekey="5aaa3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMnaa", filewrite=False) + + # Good privatekey + content = site.content_manager.sign(inner_path, privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) + content_old = site.content_manager.contents[inner_path] # Content before the sign + assert not content_old == content # Timestamp changed + assert site.address in content["signs"] # Used the site's private key to sign + if inner_path == "content.json": + assert len(content["files"]) == 17 + elif inner_path == "data/test-include/content.json": + assert len(content["files"]) == 1 + elif inner_path == "data/users/content.json": + assert len(content["files"]) == 0 + + # Everything should be same as before except the modified timestamp and the signs + assert ( + {key: val for key, val in content_old.items() if key not in ["modified", "signs", "sign"]} + == + {key: val for key, val in content.items() if key not in ["modified", "signs", "sign"]} + ) + + def testSignOptionalFiles(self, site): + site.content_manager.contents["content.json"]["optional"] = "((data/img/zero.*))" + content_optional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) + + del site.content_manager.contents["content.json"]["optional"] + content_nooptional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) + + assert len(content_nooptional.get("files_optional", {})) == 0 + assert len(content_optional["files_optional"]) > 0 + assert len(content_nooptional["files"]) > len(content_optional["files"]) + + def testFileInfo(self, site): + assert "sha512" in site.content_manager.getFileInfo("index.html") + assert not site.content_manager.getFileInfo("notexist") + + # Optional file + file_info_optional = site.content_manager.getFileInfo("data/optional.txt") + assert "sha512" in file_info_optional + assert file_info_optional["optional"] is True + + # Not exists yet user content.json + assert "cert_signers" in site.content_manager.getFileInfo("data/users/unknown/content.json") + + # Optional user file + file_info_optional = site.content_manager.getFileInfo("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + assert "sha512" in file_info_optional + assert file_info_optional["optional"] is True diff --git a/src/Test/TestContentUser.py b/src/Test/TestContentUser.py index 238a65e1..ca2184fb 100644 --- a/src/Test/TestContentUser.py +++ b/src/Test/TestContentUser.py @@ -50,6 +50,66 @@ class TestUserContent: rules = site.content_manager.getRules("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) assert rules is False + def testVerify(self, site): + privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT + user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json" + data_dict = site.content_manager.contents[user_inner_path] + users_content = site.content_manager.contents["data/users/content.json"] + + data = StringIO(json.dumps(data_dict)) + assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + + # Test max size exception by setting allowed to 0 + rules = site.content_manager.getRules(user_inner_path, data_dict) + assert rules["max_size"] == 10000 + assert users_content["user_contents"]["permission_rules"][".*"]["max_size"] == 10000 + + users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 0 + rules = site.content_manager.getRules(user_inner_path, data_dict) + assert rules["max_size"] == 0 + data = StringIO(json.dumps(data_dict)) + assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 10000 # Reset + + # Test max optional size exception + # 1 MB gif = Allowed + data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024 + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = StringIO(json.dumps(data_dict)) + assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + + # 100 MB gif = Not allowed + data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 100 * 1024 * 1024 + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = StringIO(json.dumps(data_dict)) + assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024 # Reset + + # hello.exe = Not allowed + data_dict["files_optional"]["hello.exe"] = data_dict["files_optional"]["peanut-butter-jelly-time.gif"] + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = StringIO(json.dumps(data_dict)) + assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + del data_dict["files_optional"]["hello.exe"] # Reset + + # Includes not allowed in user content + data_dict["includes"] = { "other.json": { } } + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = StringIO(json.dumps(data_dict)) + assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + def testCert(self, site): # user_addr = "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C" user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A" diff --git a/src/Test/TestHelper.py b/src/Test/TestHelper.py index c017dea9..3b4a196f 100644 --- a/src/Test/TestHelper.py +++ b/src/Test/TestHelper.py @@ -1,17 +1,39 @@ -class Spy: - def __init__(self, obj, func_name): - self.obj = obj - self.func_name = func_name - self.func_original = getattr(self.obj, func_name) - self.calls = [] +import socket - def __enter__(self, *args, **kwargs): - def loggedFunc(cls, *args, **kwags): - print "Logging", self, args, kwargs - self.calls.append(args) - return self.func_original(cls, *args, **kwargs) - setattr(self.obj, self.func_name, loggedFunc) - return self.calls +import pytest +from util import helper - def __exit__(self, *args, **kwargs): - setattr(self.obj, self.func_name, self.func_original) \ No newline at end of file + +@pytest.mark.usefixtures("resetSettings") +class TestHelper: + def testShellquote(self): + assert helper.shellquote("hel'lo") == "\"hel'lo\"" # Allow ' + assert helper.shellquote('hel"lo') == '"hello"' # Remove " + assert helper.shellquote("hel'lo", 'hel"lo') == ('"hel\'lo"', '"hello"') + + def testPackAddress(self): + assert len(helper.packAddress("1.1.1.1", 1)) == 6 + assert helper.unpackAddress(helper.packAddress("1.1.1.1", 1)) == ("1.1.1.1", 1) + + with pytest.raises(socket.error): + helper.packAddress("999.1.1.1", 1) + + with pytest.raises(socket.error): + helper.unpackAddress("X") + + def testGetDirname(self): + assert helper.getDirname("data/users/content.json") == "data/users/" + assert helper.getDirname("data/users") == "data/" + assert helper.getDirname("") == "" + assert helper.getDirname("content.json") == "" + assert helper.getDirname("data/users/") == "data/users/" + assert helper.getDirname("/data/users/content.json") == "/data/users/" + + + def testGetFilename(self): + assert helper.getFilename("data/users/content.json") == "content.json" + assert helper.getFilename("data/users") == "users" + assert helper.getFilename("") == "" + assert helper.getFilename("content.json") == "content.json" + assert helper.getFilename("data/users/") == "" + assert helper.getFilename("/data/users/content.json") == "content.json" \ No newline at end of file diff --git a/src/Test/TestSite.py b/src/Test/TestSite.py index 3aaad8e5..5aa14069 100644 --- a/src/Test/TestSite.py +++ b/src/Test/TestSite.py @@ -62,36 +62,3 @@ class TestSite: assert new_site.address in SiteManager.site_manager.sites SiteManager.site_manager.delete(new_site.address) assert new_site.address not in SiteManager.site_manager.sites - - @pytest.mark.parametrize("inner_path", ["content.json", "data/test_include/content.json", "data/users/content.json"]) - def testSign(self, site, inner_path): - # Bad privatekey - assert not site.content_manager.sign(inner_path, privatekey="5aaa3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMnaa", filewrite=False) - - # Good privatekey - content = site.content_manager.sign(inner_path, privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) - content_old = site.content_manager.contents[inner_path] # Content before the sign - assert not content_old == content # Timestamp changed - assert site.address in content["signs"] # Used the site's private key to sign - if inner_path == "content.json": - assert len(content["files"]) == 24 - elif inner_path == "data/test-include/content.json": - assert len(content["files"]) == 1 - elif inner_path == "data/users/content.json": - assert len(content["files"]) == 0 - - # Everything should be same as before except the modified timestamp and the signs - assert ( - {key: val for key, val in content_old.items() if key not in ["modified", "signs", "sign"]} - == - {key: val for key, val in content.items() if key not in ["modified", "signs", "sign"]} - ) - - def testSignOptionalFiles(self, site): - site.content_manager.contents["content.json"]["optional"] = "((data/img/zero.*))" - content_optional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) - - del site.content_manager.contents["content.json"]["optional"] - content_nooptional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) - - assert len(content_nooptional["files"]) > len(content_optional["files"]) diff --git a/src/Test/TestSiteDownload.py b/src/Test/TestSiteDownload.py new file mode 100644 index 00000000..86e5f4d8 --- /dev/null +++ b/src/Test/TestSiteDownload.py @@ -0,0 +1,63 @@ +import pytest +import mock +import time + +from Connection import ConnectionServer +from Config import config +from Site import Site +from File import FileRequest +import Spy + +@pytest.mark.usefixtures("resetTempSettings") +@pytest.mark.usefixtures("resetSettings") +class TestWorker: + def testDownload(self, file_server, site, site_temp): + client = ConnectionServer("127.0.0.1", 1545) + assert site.storage.directory == config.data_dir + "/" + site.address + assert site_temp.storage.directory == config.data_dir + "-temp/" + site.address + + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + site_temp.connection_server = client + site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net + + site_temp.addPeer("127.0.0.1", 1544) + with Spy.Spy(FileRequest, "route") as requests: + def boostRequest(inner_path): + # I really want these file + if inner_path == "index.html": + print "needFile" + site_temp.needFile("data/img/multiuser.png", priority=9, blocking=False) + site_temp.needFile("data/img/direct_domains.png", priority=10, blocking=False) + site_temp.onFileDone.append(boostRequest) + site_temp.download(blind_includes=True).join(timeout=5) + file_requests = [request[2]["inner_path"] for request in requests if request[0] in ("getFile", "streamFile")] + # Test priority + assert file_requests[0:2] == ["content.json", "index.html"] # Must-have files + assert file_requests[2:4] == ["data/img/direct_domains.png", "data/img/multiuser.png"] # Directly requested files + assert file_requests[4:6] == ["css/all.css", "js/all.js"] # Important assets + assert file_requests[6] == "dbschema.json" # Database map + assert "-default" in file_requests[-1] # Put default files for cloning to the end + + # Check files + bad_files = site_temp.storage.verifyFiles(quick_check=True) + + # -1 because data/users/1J6... user has invalid cert + assert len(site_temp.content_manager.contents) == len(site.content_manager.contents) - 1 + assert not bad_files + + # Optional file + assert not site_temp.storage.isFile("data/optional.txt") + assert site.storage.isFile("data/optional.txt") + site_temp.needFile("data/optional.txt") + assert site_temp.storage.isFile("data/optional.txt") + + # Optional user file + assert not site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + site_temp.needFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + assert site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + + assert site_temp.storage.deleteFiles() diff --git a/src/Test/TestSiteStorage.py b/src/Test/TestSiteStorage.py index 91eb43c4..b4d45468 100644 --- a/src/Test/TestSiteStorage.py +++ b/src/Test/TestSiteStorage.py @@ -1,15 +1,13 @@ -import shutil -import os - import pytest -from Site import SiteManager @pytest.mark.usefixtures("resetSettings") class TestSiteStorage: def testList(self, site): - list_root = list(site.storage.list("")) - assert "content.json" in list_root - assert "css/all.css" in list_root + # Rootdir + list_root = list(site.storage.list("")) + assert "content.json" in list_root + assert "css/all.css" in list_root + # Subdir assert list(site.storage.list("data-default")) == ["data.json", "users/content-default.json"] diff --git a/src/Test/conftest.py b/src/Test/conftest.py index 441e3bdb..751700eb 100644 --- a/src/Test/conftest.py +++ b/src/Test/conftest.py @@ -22,6 +22,7 @@ from Config import config config.argv = ["none"] # Dont pass any argv to config parser config.parse() config.data_dir = "src/Test/testdata" # Use test data for unittests +config.debug_socket = True # Use test data for unittests logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) from Site import Site @@ -76,7 +77,7 @@ def resetTempSettings(request): request.addfinalizer(cleanup) -@pytest.fixture(scope="session") +@pytest.fixture() def site(): site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") return site diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json index fdc49a19..dd20db92 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json @@ -1,133 +1,136 @@ { - "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", - "background-color": "white", - "description": "Blogging platform Demo", - "domain": "Blog.ZeroNetwork.bit", + "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", + "background-color": "white", + "description": "Blogging platform Demo", + "domain": "Blog.ZeroNetwork.bit", "files": { "css/all.css": { - "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", + "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", "size": 112710 - }, + }, "data-default/data.json": { - "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", + "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", "size": 196 - }, + }, "data-default/users/content-default.json": { - "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", + "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", "size": 679 - }, + }, "data/data.json": { - "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", + "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", "size": 31126 - }, + }, "data/img/autoupdate.png": { - "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", + "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", "size": 24460 - }, + }, "data/img/direct_domains.png": { - "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", + "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", "size": 16185 - }, + }, "data/img/domain.png": { - "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", + "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", "size": 11881 - }, + }, "data/img/memory.png": { - "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", + "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", "size": 12775 - }, + }, "data/img/multiuser.png": { - "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", + "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", "size": 29480 - }, + }, "data/img/progressbar.png": { - "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", + "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", "size": 13294 - }, + }, "data/img/slides.png": { - "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", + "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", "size": 14439 - }, + }, "data/img/slots_memory.png": { - "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", + "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", "size": 9488 - }, + }, "data/img/trayicon.png": { - "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", + "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", "size": 19040 - }, - "data/img/zeroblog-comments.png": { - "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", - "size": 24001 - }, - "data/img/zeroid.png": { - "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", - "size": 18875 - }, - "data/img/zeroname.png": { - "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", - "size": 36031 - }, - "data/img/zerotalk-mark.png": { - "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", - "size": 44862 - }, - "data/img/zerotalk-upvote.png": { - "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", - "size": 41092 - }, - "data/img/zerotalk.png": { - "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", - "size": 26606 - }, - "data/test_include/data.json": { - "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", - "size": 505 - }, + }, "dbschema.json": { - "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", + "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", "size": 1529 - }, + }, "img/loading.gif": { - "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", + "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", "size": 723 - }, + }, "index.html": { - "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", + "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", "size": 4667 - }, + }, "js/all.js": { - "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", + "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", "size": 201178 } - }, - "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)", + }, + "files_optional": { + "data/img/zeroblog-comments.png": { + "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", + "size": 24001 + }, + "data/img/zeroid.png": { + "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", + "size": 18875 + }, + "data/img/zeroname.png": { + "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", + "size": 36031 + }, + "data/img/zerotalk-mark.png": { + "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", + "size": 44862 + }, + "data/img/zerotalk-upvote.png": { + "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", + "size": 41092 + }, + "data/img/zerotalk.png": { + "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", + "size": 26606 + }, + "data/optional.txt": { + "sha512": "c6f81db0e9f8206c971c9e5826e3ba823ffbb1a3a900f8047652a8bf78ea98fd", + "size": 6 + } + }, + "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*|data/test_include/.*)", "includes": { "data/test_include/content.json": { - "added": 1424976057, - "files_allowed": "data.json", - "includes_allowed": false, - "max_size": 20000, - "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], - "signers_required": 1, - "user_id": 47, + "added": 1424976057, + "files_allowed": "data.json", + "includes_allowed": false, + "max_size": 20000, + "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], + "signers_required": 1, + "user_id": 47, "user_name": "test" - }, + }, "data/users/content.json": { - "signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ], + "signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ], "signers_required": 1 } - }, - "modified": 1443393859.801, + }, + "modified": 1443645832.748, + "optional": "(data/img/zero.*|data/optional.txt)", "sign": [ - 30041653970398729892154852118727733790145614202537425646336077462070348808967, - 96823925597554846684463773054016176426938620086211253074026312396122955360853 - ], - "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", + 33155653220731268227776289017011639520872180216646876377169089096034035969487, + 36744504416132878244552522451563313660303086381031784548929582417244124447603 + ], + "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", "signs": { - "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HKBBvaOi1v20ZuSVORtD4bBRRgf/85QDVy4HaaPX3fFDAKYmvUWK+Jbp3yIGElMmPoO2+YljFLyromAoEwWd6Eg=" - }, - "signs_required": 1, - "title": "ZeroBlog", + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HP+qomOGy0hTFX1HjHv8iQIL6E22qNynb+IijEblL2lm8SgsyiOxKGaVkD6/eE6xYGeYHSnhSii2Gw/04z3okNM=" + }, + "signs_required": 1, + "title": "ZeroBlog", "zeronet_version": "0.3.2" } \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/optional.txt b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/optional.txt new file mode 100644 index 00000000..3462721f --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/optional.txt @@ -0,0 +1 @@ +hello! \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json index 9c0812d0..3bb3929f 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json @@ -1,15 +1,22 @@ { - "cert_auth_type": "web", - "cert_sign": "HBsTrjTmv+zD1iY93tSci8n9DqdEtYwzxJmRppn4/b+RYktcANGm5tXPOb+Duw3AJcgWDcGUvQVgN1D9QAwIlCw=", - "cert_user_id": "toruser@zeroid.bit", - "files": { - "data.json": { - "sha512": "4868b5e6d70a55d137db71c2e276bda80437e0235ac670962acc238071296b45", - "size": 168 - } - }, - "modified": 1432491109.11, - "signs": { - "1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9": "HMy7ZwwqE0Sk8O+5hTx/ejFW6KtIDbID6fGblCodUTpz4mJZ5GwApBHSVLMYL43vvGT/vKZOiQoJ5tQTeFVbbkk=" + "cert_auth_type": "web", + "cert_sign": "HBsTrjTmv+zD1iY93tSci8n9DqdEtYwzxJmRppn4/b+RYktcANGm5tXPOb+Duw3AJcgWDcGUvQVgN1D9QAwIlCw=", + "cert_user_id": "toruser@zeroid.bit", + "files": { + "data.json": { + "sha512": "4868b5e6d70a55d137db71c2e276bda80437e0235ac670962acc238071296b45", + "size": 168 } + }, + "files_optional": { + "peanut-butter-jelly-time.gif": { + "sha512": "a238fd27bda2a06f07f9f246954b34dcf82e6472aebdecc2c5dc1f01a50721ef", + "size": 1606 + } + }, + "modified": 1443645834.763, + "optional": ".*\\.(jpg|png|gif)", + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HD+/5Jmew6BotfeWsfpTFKXUVIY5MyjKQx5KRnT6WO0nMBLxaI6/sTb+6ZXq0tXjXNkmlt36/UICYQcYQjCRhkY=" + } } \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif new file mode 100644 index 0000000000000000000000000000000000000000..54c69d1c8f509f0df86c2a1b387e7104bc040844 GIT binary patch literal 1606 zcmZvbc~H`67{`Bwh~g2Jt_qXlU}ogS55#i~#9Q-7T}?AJ70t6e50XeV^y!>Fr_f5NiTpfOi1! z^;-d70bpqfkVpVL75KkQzH0cphXhl|f$pG{4P@6=*W{CmIvjuj7T;cc8wVgQwDHw1 zHVfW8llJSkrMRf~1&`toD&cfDQeU~wlC{M$C2CD6>~GoQ-+==vSaENEQ#VU+(VHy^ z@yI13@i$RvQcThHjkL1au!jZ_YZX-AzY=sgv*c`ewJA6HNC4g26Xu-D<76TjH0Eg* zm!5Cu;(&$b{Q%|J2dPS-Qe?An1G-sm*hy~e>@tvOU36my*kVctI7RT#iL73n5~02m zciT3%0$KCnnJgZmV&ED#AWM4Kml~%~0<#ORw291o>3z$1KzG+nL?zVi4$VxiBjG+~ zIZlJ^!D*vQm82#TxD=IZd zL7^~clrynLB&O`GA^Y&9wQe-IeSM(Jq>&`;M6|lm!OPhp9)c3#m_neK$nMh&*V&O= zu|=LZGw-csQuUJl_m>J>jFEy&wyC9xx&s;42Gj=^_pd54{{cw7v1)MsbYRd&l4IFQ z9Qgc%f042c0h|AfhL*k=&D3S`*>zqTLW7qBW(`jaRVd@7O#P3W2`L~v8BwTe!{#_R zncJ495+DgeleC&2IVII$LaBAFsG_M^zNcPN*V@&MJ=_b%%i5hY+s|l>j={>x#>OYE za!-$)%SjrH@C!da;N}sTvvS90?w4WMj$sDBuR(;fwWg^3s`tF;p1Y>8p&PvyKDd)JY142EFyG6+F%2c9g0|$2Hm@MC4}4^42ll@<8NXs4BjQ))R@D&M1++ zoJCD!wo=<%rDAYUd)P0J-=NRU=X+I;Uz2|#x!^Q_VF!xs^!AGrim~)dGm|F zHXfa%N#3Y^z-c#ik{w!dIm_$m;T~26nmYDr^RG-()YL)DVpF!7J61A(;4YGmHHN-; zq4YI^PkF>%!rV)($O*^5kSdDzlBiRSZ*~{a7+GKnfkjiz2lcF-sQiQ0wpCzF2A1za z$PizsJtNhIG&kDGH9LI+b%qMSYVnBRkGRz%2EA)=Y~@v*F)dJAu;Br*J5s%qVwuM z8r*t%CNF=cVr9*J>=3cVdi9lxj5!$9NZ!z(XI$7FLh3*%%GkEYgjd%=S^<&+16?dm z@VEu1O05c*Uii{9Cjo(ks)f)<0;{7nmGTyuxG}PmM0csKgZ20K4h<7WN6~$_9$kH^ V`Bai&kQ*v^2ASNs*rpCR{RfJmfm#3n literal 0 HcmV?d00001 diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json index 9c0812d0..ada2c7ac 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json @@ -1,15 +1,15 @@ { - "cert_auth_type": "web", - "cert_sign": "HBsTrjTmv+zD1iY93tSci8n9DqdEtYwzxJmRppn4/b+RYktcANGm5tXPOb+Duw3AJcgWDcGUvQVgN1D9QAwIlCw=", - "cert_user_id": "toruser@zeroid.bit", - "files": { - "data.json": { - "sha512": "4868b5e6d70a55d137db71c2e276bda80437e0235ac670962acc238071296b45", - "size": 168 - } - }, - "modified": 1432491109.11, - "signs": { - "1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9": "HMy7ZwwqE0Sk8O+5hTx/ejFW6KtIDbID6fGblCodUTpz4mJZ5GwApBHSVLMYL43vvGT/vKZOiQoJ5tQTeFVbbkk=" + "cert_auth_type": "web", + "cert_sign": "HBsTrjTmv+zD1iY93tSci8n9DqdEtYwzxJmRppn4/b+RYktcANGm5tXPOb+Duw3AJcgWDcGUvQVgN1D9QAwIlCw=", + "cert_user_id": "toruser@zeroid.bit", + "files": { + "data.json": { + "sha512": "4868b5e6d70a55d137db71c2e276bda80437e0235ac670962acc238071296b45", + "size": 168 } + }, + "modified": 1443645835.157, + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HNIXDWV1kpqKtsJ+yrNKLvks/FDYIpVmx7xgkXPJ6NZiajCMHrgEwLH9QRiq6rs3nOCs0P08eRhlgZLvC+3U6ps=" + } } \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json index dabbe5f8..b3894239 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json @@ -1,9 +1,9 @@ { "files": {}, "ignore": ".*", - "modified": 1443088330.941, + "modified": 1443645833.247, "signs": { - "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G/YCfchtojDA7EjXk5Xa6af5EaEME14LDAvVE9P8PCDb2ncWN79ZTMsczAx7N3HYyM9Vdqn+8or4hh28z4ITKqU=" + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G2EcaEKdzzfbpRITDcaBajNwjaIIJW3zp1YQGIMJcxfw3tLnn6uv/goImvbzvuTXKkl5fQKmBowK2Bg1xXJ3078=" }, "user_contents": { "cert_signers": { @@ -12,7 +12,9 @@ "permission_rules": { ".*": { "files_allowed": "data.json", + "files_allowed_optional": ".*\\.(png|jpg|gif)", "max_size": 10000, + "max_size_optional": 10000000, "signers": [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" ] }, "bitid/.*@zeroid.bit": { "max_size": 40000 }, diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py index 06d438e6..f4ee0e51 100644 --- a/src/Worker/WorkerManager.py +++ b/src/Worker/WorkerManager.py @@ -83,7 +83,9 @@ class WorkerManager: elif inner_path.endswith(".css"): priority += 5 # boost css files priority elif inner_path.endswith(".js"): - priority += 3 # boost js files priority + priority += 4 # boost js files priority + elif inner_path.endswith("dbschema.json"): + priority += 3 # boost database specification elif inner_path.endswith("content.json"): priority += 1 # boost included content.json files priority a bit elif inner_path.endswith(".json"): diff --git a/src/util/helper.py b/src/util/helper.py index 2d8c4f33..b78d06ef 100644 --- a/src/util/helper.py +++ b/src/util/helper.py @@ -1,6 +1,7 @@ import os import socket import struct +import re def atomicWrite(dest, content, mode="w"): @@ -16,10 +17,27 @@ def shellquote(*args): else: return tuple(['"%s"' % arg.replace('"', "") for arg in args]) + # ip, port to packed 6byte format def packAddress(ip, port): return socket.inet_aton(ip) + struct.pack("H", port) + # From 6byte format to ip, port def unpackAddress(packed): - return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0] \ No newline at end of file + return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0] + + +# Get dir from file +# Return: data/site/content.json -> data/site +def getDirname(path): + file_dir = re.sub("[^/]*?$", "", path).rstrip("/") + if file_dir: + file_dir += "/" # Add / at end if its not the root + return file_dir + + +# Get dir from file +# Return: data/site/content.json -> content.json +def getFilename(path): + return re.sub("^.*/", "", path)