diff --git a/requirements.txt b/requirements.txt index 61555a22..f60dc30b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -gevent==1.0.1 -msgpack-python==0.4.4 +gevent>=1.0.1 +msgpack-python>=0.4.4 diff --git a/src/Config.py b/src/Config.py index 62c76faa..6bd44974 100644 --- a/src/Config.py +++ b/src/Config.py @@ -8,7 +8,7 @@ class Config(object): def __init__(self, argv): self.version = "0.3.2" - self.rev = 465 + self.rev = 467 self.argv = argv self.action = None self.createParser() diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index 3fee5de4..094a11b2 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -9,7 +9,7 @@ import gevent from Debug import Debug from Crypt import CryptHash from Config import config - +from util import helper class ContentManager(object): @@ -26,8 +26,8 @@ class ContentManager(object): content_inner_path = content_inner_path.strip("/") # Remove / from begning old_content = self.contents.get(content_inner_path) content_path = self.site.storage.getPath(content_inner_path) - content_dir = self.toDir(self.site.storage.getPath(content_inner_path)) - content_inner_dir = self.toDir(content_inner_path) + content_dir = helper.getDirname(self.site.storage.getPath(content_inner_path)) + content_inner_dir = helper.getDirname(content_inner_path) if os.path.isfile(content_path): try: @@ -140,16 +140,29 @@ class ContentManager(object): while True: content_inner_path = "%s/content.json" % "/".join(dirs) content = self.contents.get(content_inner_path.strip("/")) - if content and "files" in content: # Check if content.json exists + + # Check in files + if content and "files" in content: back = content["files"].get("/".join(inner_path_parts)) if back: back["content_inner_path"] = content_inner_path + back["optional"] = False return back - if content and "user_contents" in content: # User dir + # Check in optional files + if content and "files_optional" in content: # Check if file in this content.json + back = content["files_optional"].get("/".join(inner_path_parts)) + if back: + back["content_inner_path"] = content_inner_path + back["optional"] = True + return back + + # Return the rules if user dir + if content and "user_contents" in content: back = content["user_contents"] # Content.json is in the users dir back["content_inner_path"] = re.sub("(.*)/.*?$", "\\1/content.json", inner_path) + back["optional"] = None return back # No inner path in this dir, lets try the parent dir @@ -234,6 +247,7 @@ class ContentManager(object): rules["signers"] = [] rules["signers"].append(user_address) # Add user as valid signer rules["user_address"] = user_address + rules["includes_allowed"] = False return rules @@ -243,7 +257,7 @@ class ContentManager(object): files_optional_node = {} for file_relative_path in self.site.storage.list(dir_inner_path): - file_name = self.toFilename(file_relative_path) + file_name = helper.getFilename(file_relative_path) ignored = optional = False if file_name == "content.json": @@ -283,12 +297,12 @@ class ContentManager(object): if extend: content.update(extend) # Add custom fields - directory = self.toDir(self.site.storage.getPath(inner_path)) - inner_directory = self.toDir(inner_path) + directory = helper.getDirname(self.site.storage.getPath(inner_path)) + inner_directory = helper.getDirname(inner_path) self.log.info("Opening site data directory: %s..." % directory) changed_files = [inner_path] - files_node, files_optional_node = self.hashFiles(self.toDir(inner_path), content.get("ignore"), content.get("optional")) + files_node, files_optional_node = self.hashFiles(helper.getDirname(inner_path), content.get("ignore"), content.get("optional")) # Find changed files files_merged = files_node.copy() @@ -310,13 +324,17 @@ class ContentManager(object): new_content = content.copy() # Create a copy of current content.json new_content["files"] = files_node # Add files sha512 hash if files_optional_node: - new_content["files_optional_node"] = files_optional_node + new_content["files_optional"] = files_optional_node + elif "files_optional" in new_content: + del new_content["files_optional"] + new_content["modified"] = time.time() # Add timestamp if inner_path == "content.json": new_content["address"] = self.site.address new_content["zeronet_version"] = config.version new_content["signs_required"] = content.get("signs_required", 1) + # Verify private key from Crypt import CryptBitcoin self.log.info("Verifying private key...") privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey) @@ -409,6 +427,7 @@ class ContentManager(object): # Return: True or False def verifyContent(self, inner_path, content): content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content + content_size_optional = sum([file["size"] for file in content.get("files_optional", {}).values()]) site_size = self.getTotalSize(ignore=inner_path) + content_size # Site size without old content if site_size > self.site.settings.get("size", 0): self.site.settings["size"] = site_size # Save to settings if larger @@ -433,23 +452,34 @@ class ContentManager(object): return False # Check include size limit - if rules.get("max_size"): # Include size limit + if rules.get("max_size") is not None: # Include size limit if content_size > rules["max_size"]: self.log.error("%s: Include too large %s > %s" % (inner_path, content_size, rules["max_size"])) return False - # Check if content includes allowed - if rules.get("includes_allowed") is False and content.get("includes"): - self.log.error("%s: Includes not allowed" % inner_path) - return False # Includes not allowed + if rules.get("max_size_optional") is not None: # Include optional files limit + if content_size_optional > rules["max_size_optional"]: + self.log.error("%s: Include optional files too large %s > %s" % (inner_path, content_size_optional, rules["max_size_optional"])) + return False # Filename limit if rules.get("files_allowed"): for file_inner_path in content["files"].keys(): if not re.match("^%s$" % rules["files_allowed"], file_inner_path): - self.log.error("%s: File not allowed" % file_inner_path) + self.log.error("%s %s: File not allowed" % (inner_path, file_inner_path)) return False + if rules.get("files_allowed_optional"): + for file_inner_path in content.get("files_optional", {}).keys(): + if not re.match("^%s$" % rules["files_allowed_optional"], file_inner_path): + self.log.error("%s %s: Optional file not allowed" % (inner_path, file_inner_path)) + return False + + # Check if content includes allowed + if rules.get("includes_allowed") is False and content.get("includes"): + self.log.error("%s: Includes not allowed" % inner_path) + return False # Includes not allowed + return True # All good # Verify file validity @@ -507,7 +537,7 @@ class ContentManager(object): valid_signs += CryptBitcoin.verify(sign_content, address, signs[address]) if valid_signs >= signs_required: break # Break if we has enough signs - + self.log.debug("%s: Valid signs: %s/%s" % (inner_path, valid_signs, signs_required)) return valid_signs >= signs_required else: # Old style signing return CryptBitcoin.verify(sign_content, self.site.address, sign) @@ -537,19 +567,6 @@ class ContentManager(object): self.log.error("File not in content.json: %s" % inner_path) return False - # Get dir from file - # Return: data/site/content.json -> data/site - def toDir(self, inner_path): - file_dir = re.sub("[^/]*?$", "", inner_path).strip("/") - if file_dir: - file_dir += "/" # Add / at end if its not the root - return file_dir - - # Get dir from file - # Return: data/site/content.json -> data/site - def toFilename(self, inner_path): - return re.sub("^.*/", "", inner_path) - if __name__ == "__main__": def testSign(): diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 77b689b7..af3ca228 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -1,6 +1,8 @@ import logging -import gevent import time +import array + +import gevent from cStringIO import StringIO from Debug import Debug @@ -14,8 +16,8 @@ if config.use_tempfiles: # Communicate remote peers class Peer(object): __slots__ = ( - "ip", "port", "site", "key", "connection", "last_found", "last_response", - "last_ping", "added", "connection_error", "hash_failed", "download_bytes", "download_time" + "ip", "port", "site", "key", "connection", "last_found", "last_response", "last_ping", "last_hashfield", + "hashfield", "added", "connection_error", "hash_failed", "download_bytes", "download_time" ) def __init__(self, ip, port, site=None): @@ -25,6 +27,8 @@ class Peer(object): self.key = "%s:%s" % (ip, port) self.connection = None + self.hashfield = array.array("H") # Got optional files hash_id + self.last_hashfield = None # Last time hashfiled downloaded self.last_found = time.time() # Time of last found in the torrent tracker self.last_response = None # Time of last successful response from peer self.last_ping = None # Last response time for ping @@ -230,6 +234,34 @@ class Peer(object): if self.connection: self.connection.close() + # - HASHFIELD - + + def updateHashfield(self, force=False): + # Don't update hashfield again in 15 min + if self.last_hashfield and time.time() - self.last_hashfield > 60 * 15 and not force: + return False + + response = self.request("getHashfield", {"site": self.site.address}) + if not response or "error" in response: + return False + self.last_hashfield = time.time() + self.hashfield = response["hashfield"] + + return self.hashfield + + def setHashfield(self, hashfield_dump): + self.hashfield.fromstring(hashfield_dump) + + def hasHash(self, hash_id): + return hash_id in self.hashfield + + # Return: ["ip:port", "ip:port",...] + def findHash(self, hash_id): + response = self.request("findHash", {"site": self.site.address, "hash_id": hash_id}) + if not response or "error" in response: + return False + return [helper.unpackAddress(peer) for peer in response["peers"]] + # - EVENTS - # On connection error diff --git a/src/Site/Site.py b/src/Site/Site.py index 3a6f84a0..cdaad9d8 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -112,7 +112,7 @@ class Site: s = time.time() self.log.debug("Downloading %s..." % inner_path) found = self.needFile(inner_path, update=self.bad_files.get(inner_path)) - content_inner_dir = self.content_manager.toDir(inner_path) + content_inner_dir = helper.getDirname(inner_path) if not found: self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications)) if check_modifications: # Download failed, but check modifications if its succed later @@ -386,7 +386,7 @@ class Site: # Copy files for content_inner_path, content in self.content_manager.contents.items(): for file_relative_path in sorted(content["files"].keys()): - file_inner_path = self.content_manager.toDir(content_inner_path) + file_relative_path # Relative to content.json + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to content.json file_inner_path = file_inner_path.strip("/") # Strip leading / if file_inner_path.split("/")[0] in default_dirs: # Dont copy directories that has -default postfixed alternative self.log.debug("[SKIP] %s (has default alternative)" % file_inner_path) diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index 7856fad8..5b87f306 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -3,7 +3,6 @@ import re import shutil import json import time -import sys import sqlite3 import gevent.event @@ -11,6 +10,7 @@ import gevent.event from Db import Db from Debug import Debug from Config import config +from util import helper class SiteStorage: @@ -98,7 +98,7 @@ class SiteStorage: for file_relative_path in content["files"].keys(): if not file_relative_path.endswith(".json"): continue # We only interesed in json files - content_inner_path_dir = self.site.content_manager.toDir(content_inner_path) # Content.json dir relative to site + content_inner_path_dir = helper.getDirname(content_inner_path) # Content.json dir relative to site file_inner_path = content_inner_path_dir + file_relative_path # File Relative to site dir file_inner_path = file_inner_path.strip("/") # Strip leading / file_path = self.getPath(file_inner_path) @@ -170,7 +170,6 @@ class SiteStorage: else: yield file_name - # Site content updated def onUpdated(self, inner_path): file_path = self.getPath(inner_path) @@ -255,7 +254,7 @@ class SiteStorage: self.log.debug("[MISSING] %s" % content_inner_path) bad_files.append(content_inner_path) for file_relative_path in content["files"].keys(): - file_inner_path = self.site.content_manager.toDir(content_inner_path) + file_relative_path # Relative to site dir + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir file_inner_path = file_inner_path.strip("/") # Strip leading / file_path = self.getPath(file_inner_path) if not os.path.isfile(file_path): @@ -304,8 +303,13 @@ class SiteStorage: files = [] # Get filenames for content_inner_path, content in self.site.content_manager.contents.items(): files.append(content_inner_path) - for file_relative_path in content["files"].keys(): - file_inner_path = self.site.content_manager.toDir(content_inner_path) + file_relative_path # Relative to site dir + # Add normal files + for file_relative_path in content.get("files", {}).keys(): + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir + files.append(file_inner_path) + # Add optional files + for file_relative_path in content.get("files_optional", {}).keys(): + file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir files.append(file_inner_path) for inner_path in files: diff --git a/src/Test/Spy.py b/src/Test/Spy.py new file mode 100644 index 00000000..c017dea9 --- /dev/null +++ b/src/Test/Spy.py @@ -0,0 +1,17 @@ +class Spy: + def __init__(self, obj, func_name): + self.obj = obj + self.func_name = func_name + self.func_original = getattr(self.obj, func_name) + self.calls = [] + + def __enter__(self, *args, **kwargs): + def loggedFunc(cls, *args, **kwags): + print "Logging", self, args, kwargs + self.calls.append(args) + return self.func_original(cls, *args, **kwargs) + setattr(self.obj, self.func_name, loggedFunc) + return self.calls + + def __exit__(self, *args, **kwargs): + setattr(self.obj, self.func_name, self.func_original) \ No newline at end of file diff --git a/src/Test/TestContent.py b/src/Test/TestContent.py index c7130a5b..5303287e 100644 --- a/src/Test/TestContent.py +++ b/src/Test/TestContent.py @@ -9,7 +9,7 @@ from Crypt import CryptBitcoin @pytest.mark.usefixtures("resetSettings") class TestContent: - def testIncludes(self, site): + def testInclude(self, site): # Rules defined in parent content.json rules = site.content_manager.getRules("data/test_include/content.json") @@ -34,7 +34,7 @@ class TestContent: # Valid signers for root content.json assert site.content_manager.getValidSigners("content.json") == ["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"] - def testLimits(self, site): + def testInlcudeLimits(self, site): privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # Data validation data_dict = { @@ -48,7 +48,7 @@ class TestContent: } # Normal data - data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) # Reset @@ -56,7 +56,7 @@ class TestContent: # Too large data_dict["files"]["data.json"]["size"] = 200000 # Emulate 2MB sized data.json - data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) assert not site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) # Reset @@ -65,7 +65,7 @@ class TestContent: # Not allowed file data_dict["files"]["notallowed.exe"] = data_dict["files"]["data.json"] - data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) assert not site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) # Reset @@ -73,6 +73,58 @@ class TestContent: del data_dict["signs"] # Should work again - data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey) } + data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) + + @pytest.mark.parametrize("inner_path", ["content.json", "data/test_include/content.json", "data/users/content.json"]) + def testSign(self, site, inner_path): + # Bad privatekey + assert not site.content_manager.sign(inner_path, privatekey="5aaa3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMnaa", filewrite=False) + + # Good privatekey + content = site.content_manager.sign(inner_path, privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) + content_old = site.content_manager.contents[inner_path] # Content before the sign + assert not content_old == content # Timestamp changed + assert site.address in content["signs"] # Used the site's private key to sign + if inner_path == "content.json": + assert len(content["files"]) == 17 + elif inner_path == "data/test-include/content.json": + assert len(content["files"]) == 1 + elif inner_path == "data/users/content.json": + assert len(content["files"]) == 0 + + # Everything should be same as before except the modified timestamp and the signs + assert ( + {key: val for key, val in content_old.items() if key not in ["modified", "signs", "sign"]} + == + {key: val for key, val in content.items() if key not in ["modified", "signs", "sign"]} + ) + + def testSignOptionalFiles(self, site): + site.content_manager.contents["content.json"]["optional"] = "((data/img/zero.*))" + content_optional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) + + del site.content_manager.contents["content.json"]["optional"] + content_nooptional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) + + assert len(content_nooptional.get("files_optional", {})) == 0 + assert len(content_optional["files_optional"]) > 0 + assert len(content_nooptional["files"]) > len(content_optional["files"]) + + def testFileInfo(self, site): + assert "sha512" in site.content_manager.getFileInfo("index.html") + assert not site.content_manager.getFileInfo("notexist") + + # Optional file + file_info_optional = site.content_manager.getFileInfo("data/optional.txt") + assert "sha512" in file_info_optional + assert file_info_optional["optional"] is True + + # Not exists yet user content.json + assert "cert_signers" in site.content_manager.getFileInfo("data/users/unknown/content.json") + + # Optional user file + file_info_optional = site.content_manager.getFileInfo("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + assert "sha512" in file_info_optional + assert file_info_optional["optional"] is True diff --git a/src/Test/TestContentUser.py b/src/Test/TestContentUser.py index 238a65e1..ca2184fb 100644 --- a/src/Test/TestContentUser.py +++ b/src/Test/TestContentUser.py @@ -50,6 +50,66 @@ class TestUserContent: rules = site.content_manager.getRules("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) assert rules is False + def testVerify(self, site): + privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT + user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json" + data_dict = site.content_manager.contents[user_inner_path] + users_content = site.content_manager.contents["data/users/content.json"] + + data = StringIO(json.dumps(data_dict)) + assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + + # Test max size exception by setting allowed to 0 + rules = site.content_manager.getRules(user_inner_path, data_dict) + assert rules["max_size"] == 10000 + assert users_content["user_contents"]["permission_rules"][".*"]["max_size"] == 10000 + + users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 0 + rules = site.content_manager.getRules(user_inner_path, data_dict) + assert rules["max_size"] == 0 + data = StringIO(json.dumps(data_dict)) + assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 10000 # Reset + + # Test max optional size exception + # 1 MB gif = Allowed + data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024 + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = StringIO(json.dumps(data_dict)) + assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + + # 100 MB gif = Not allowed + data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 100 * 1024 * 1024 + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = StringIO(json.dumps(data_dict)) + assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024 # Reset + + # hello.exe = Not allowed + data_dict["files_optional"]["hello.exe"] = data_dict["files_optional"]["peanut-butter-jelly-time.gif"] + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = StringIO(json.dumps(data_dict)) + assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + del data_dict["files_optional"]["hello.exe"] # Reset + + # Includes not allowed in user content + data_dict["includes"] = { "other.json": { } } + del data_dict["signs"] # Remove signs before signing + data_dict["signs"] = { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) + } + data = StringIO(json.dumps(data_dict)) + assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) + def testCert(self, site): # user_addr = "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C" user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A" diff --git a/src/Test/TestHelper.py b/src/Test/TestHelper.py index c017dea9..3b4a196f 100644 --- a/src/Test/TestHelper.py +++ b/src/Test/TestHelper.py @@ -1,17 +1,39 @@ -class Spy: - def __init__(self, obj, func_name): - self.obj = obj - self.func_name = func_name - self.func_original = getattr(self.obj, func_name) - self.calls = [] +import socket - def __enter__(self, *args, **kwargs): - def loggedFunc(cls, *args, **kwags): - print "Logging", self, args, kwargs - self.calls.append(args) - return self.func_original(cls, *args, **kwargs) - setattr(self.obj, self.func_name, loggedFunc) - return self.calls +import pytest +from util import helper - def __exit__(self, *args, **kwargs): - setattr(self.obj, self.func_name, self.func_original) \ No newline at end of file + +@pytest.mark.usefixtures("resetSettings") +class TestHelper: + def testShellquote(self): + assert helper.shellquote("hel'lo") == "\"hel'lo\"" # Allow ' + assert helper.shellquote('hel"lo') == '"hello"' # Remove " + assert helper.shellquote("hel'lo", 'hel"lo') == ('"hel\'lo"', '"hello"') + + def testPackAddress(self): + assert len(helper.packAddress("1.1.1.1", 1)) == 6 + assert helper.unpackAddress(helper.packAddress("1.1.1.1", 1)) == ("1.1.1.1", 1) + + with pytest.raises(socket.error): + helper.packAddress("999.1.1.1", 1) + + with pytest.raises(socket.error): + helper.unpackAddress("X") + + def testGetDirname(self): + assert helper.getDirname("data/users/content.json") == "data/users/" + assert helper.getDirname("data/users") == "data/" + assert helper.getDirname("") == "" + assert helper.getDirname("content.json") == "" + assert helper.getDirname("data/users/") == "data/users/" + assert helper.getDirname("/data/users/content.json") == "/data/users/" + + + def testGetFilename(self): + assert helper.getFilename("data/users/content.json") == "content.json" + assert helper.getFilename("data/users") == "users" + assert helper.getFilename("") == "" + assert helper.getFilename("content.json") == "content.json" + assert helper.getFilename("data/users/") == "" + assert helper.getFilename("/data/users/content.json") == "content.json" \ No newline at end of file diff --git a/src/Test/TestSite.py b/src/Test/TestSite.py index 3aaad8e5..5aa14069 100644 --- a/src/Test/TestSite.py +++ b/src/Test/TestSite.py @@ -62,36 +62,3 @@ class TestSite: assert new_site.address in SiteManager.site_manager.sites SiteManager.site_manager.delete(new_site.address) assert new_site.address not in SiteManager.site_manager.sites - - @pytest.mark.parametrize("inner_path", ["content.json", "data/test_include/content.json", "data/users/content.json"]) - def testSign(self, site, inner_path): - # Bad privatekey - assert not site.content_manager.sign(inner_path, privatekey="5aaa3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMnaa", filewrite=False) - - # Good privatekey - content = site.content_manager.sign(inner_path, privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) - content_old = site.content_manager.contents[inner_path] # Content before the sign - assert not content_old == content # Timestamp changed - assert site.address in content["signs"] # Used the site's private key to sign - if inner_path == "content.json": - assert len(content["files"]) == 24 - elif inner_path == "data/test-include/content.json": - assert len(content["files"]) == 1 - elif inner_path == "data/users/content.json": - assert len(content["files"]) == 0 - - # Everything should be same as before except the modified timestamp and the signs - assert ( - {key: val for key, val in content_old.items() if key not in ["modified", "signs", "sign"]} - == - {key: val for key, val in content.items() if key not in ["modified", "signs", "sign"]} - ) - - def testSignOptionalFiles(self, site): - site.content_manager.contents["content.json"]["optional"] = "((data/img/zero.*))" - content_optional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) - - del site.content_manager.contents["content.json"]["optional"] - content_nooptional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False) - - assert len(content_nooptional["files"]) > len(content_optional["files"]) diff --git a/src/Test/TestSiteDownload.py b/src/Test/TestSiteDownload.py new file mode 100644 index 00000000..86e5f4d8 --- /dev/null +++ b/src/Test/TestSiteDownload.py @@ -0,0 +1,63 @@ +import pytest +import mock +import time + +from Connection import ConnectionServer +from Config import config +from Site import Site +from File import FileRequest +import Spy + +@pytest.mark.usefixtures("resetTempSettings") +@pytest.mark.usefixtures("resetSettings") +class TestWorker: + def testDownload(self, file_server, site, site_temp): + client = ConnectionServer("127.0.0.1", 1545) + assert site.storage.directory == config.data_dir + "/" + site.address + assert site_temp.storage.directory == config.data_dir + "-temp/" + site.address + + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + site_temp.connection_server = client + site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net + + site_temp.addPeer("127.0.0.1", 1544) + with Spy.Spy(FileRequest, "route") as requests: + def boostRequest(inner_path): + # I really want these file + if inner_path == "index.html": + print "needFile" + site_temp.needFile("data/img/multiuser.png", priority=9, blocking=False) + site_temp.needFile("data/img/direct_domains.png", priority=10, blocking=False) + site_temp.onFileDone.append(boostRequest) + site_temp.download(blind_includes=True).join(timeout=5) + file_requests = [request[2]["inner_path"] for request in requests if request[0] in ("getFile", "streamFile")] + # Test priority + assert file_requests[0:2] == ["content.json", "index.html"] # Must-have files + assert file_requests[2:4] == ["data/img/direct_domains.png", "data/img/multiuser.png"] # Directly requested files + assert file_requests[4:6] == ["css/all.css", "js/all.js"] # Important assets + assert file_requests[6] == "dbschema.json" # Database map + assert "-default" in file_requests[-1] # Put default files for cloning to the end + + # Check files + bad_files = site_temp.storage.verifyFiles(quick_check=True) + + # -1 because data/users/1J6... user has invalid cert + assert len(site_temp.content_manager.contents) == len(site.content_manager.contents) - 1 + assert not bad_files + + # Optional file + assert not site_temp.storage.isFile("data/optional.txt") + assert site.storage.isFile("data/optional.txt") + site_temp.needFile("data/optional.txt") + assert site_temp.storage.isFile("data/optional.txt") + + # Optional user file + assert not site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + site_temp.needFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + assert site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") + + assert site_temp.storage.deleteFiles() diff --git a/src/Test/TestSiteStorage.py b/src/Test/TestSiteStorage.py index 91eb43c4..b4d45468 100644 --- a/src/Test/TestSiteStorage.py +++ b/src/Test/TestSiteStorage.py @@ -1,15 +1,13 @@ -import shutil -import os - import pytest -from Site import SiteManager @pytest.mark.usefixtures("resetSettings") class TestSiteStorage: def testList(self, site): - list_root = list(site.storage.list("")) - assert "content.json" in list_root - assert "css/all.css" in list_root + # Rootdir + list_root = list(site.storage.list("")) + assert "content.json" in list_root + assert "css/all.css" in list_root + # Subdir assert list(site.storage.list("data-default")) == ["data.json", "users/content-default.json"] diff --git a/src/Test/conftest.py b/src/Test/conftest.py index 441e3bdb..751700eb 100644 --- a/src/Test/conftest.py +++ b/src/Test/conftest.py @@ -22,6 +22,7 @@ from Config import config config.argv = ["none"] # Dont pass any argv to config parser config.parse() config.data_dir = "src/Test/testdata" # Use test data for unittests +config.debug_socket = True # Use test data for unittests logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) from Site import Site @@ -76,7 +77,7 @@ def resetTempSettings(request): request.addfinalizer(cleanup) -@pytest.fixture(scope="session") +@pytest.fixture() def site(): site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") return site diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json index fdc49a19..dd20db92 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json @@ -1,133 +1,136 @@ { - "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", - "background-color": "white", - "description": "Blogging platform Demo", - "domain": "Blog.ZeroNetwork.bit", + "address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT", + "background-color": "white", + "description": "Blogging platform Demo", + "domain": "Blog.ZeroNetwork.bit", "files": { "css/all.css": { - "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", + "sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc", "size": 112710 - }, + }, "data-default/data.json": { - "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", + "sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2", "size": 196 - }, + }, "data-default/users/content-default.json": { - "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", + "sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa", "size": 679 - }, + }, "data/data.json": { - "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", + "sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52", "size": 31126 - }, + }, "data/img/autoupdate.png": { - "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", + "sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71", "size": 24460 - }, + }, "data/img/direct_domains.png": { - "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", + "sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719", "size": 16185 - }, + }, "data/img/domain.png": { - "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", + "sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a", "size": 11881 - }, + }, "data/img/memory.png": { - "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", + "sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e", "size": 12775 - }, + }, "data/img/multiuser.png": { - "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", + "sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6", "size": 29480 - }, + }, "data/img/progressbar.png": { - "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", + "sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf", "size": 13294 - }, + }, "data/img/slides.png": { - "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", + "sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e", "size": 14439 - }, + }, "data/img/slots_memory.png": { - "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", + "sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29", "size": 9488 - }, + }, "data/img/trayicon.png": { - "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", + "sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f", "size": 19040 - }, - "data/img/zeroblog-comments.png": { - "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", - "size": 24001 - }, - "data/img/zeroid.png": { - "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", - "size": 18875 - }, - "data/img/zeroname.png": { - "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", - "size": 36031 - }, - "data/img/zerotalk-mark.png": { - "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", - "size": 44862 - }, - "data/img/zerotalk-upvote.png": { - "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", - "size": 41092 - }, - "data/img/zerotalk.png": { - "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", - "size": 26606 - }, - "data/test_include/data.json": { - "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", - "size": 505 - }, + }, "dbschema.json": { - "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", + "sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8", "size": 1529 - }, + }, "img/loading.gif": { - "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", + "sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00", "size": 723 - }, + }, "index.html": { - "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", + "sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3", "size": 4667 - }, + }, "js/all.js": { - "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", + "sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c", "size": 201178 } - }, - "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)", + }, + "files_optional": { + "data/img/zeroblog-comments.png": { + "sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4", + "size": 24001 + }, + "data/img/zeroid.png": { + "sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b", + "size": 18875 + }, + "data/img/zeroname.png": { + "sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d", + "size": 36031 + }, + "data/img/zerotalk-mark.png": { + "sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af", + "size": 44862 + }, + "data/img/zerotalk-upvote.png": { + "sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732", + "size": 41092 + }, + "data/img/zerotalk.png": { + "sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8", + "size": 26606 + }, + "data/optional.txt": { + "sha512": "c6f81db0e9f8206c971c9e5826e3ba823ffbb1a3a900f8047652a8bf78ea98fd", + "size": 6 + } + }, + "ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*|data/test_include/.*)", "includes": { "data/test_include/content.json": { - "added": 1424976057, - "files_allowed": "data.json", - "includes_allowed": false, - "max_size": 20000, - "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], - "signers_required": 1, - "user_id": 47, + "added": 1424976057, + "files_allowed": "data.json", + "includes_allowed": false, + "max_size": 20000, + "signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ], + "signers_required": 1, + "user_id": 47, "user_name": "test" - }, + }, "data/users/content.json": { - "signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ], + "signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ], "signers_required": 1 } - }, - "modified": 1443393859.801, + }, + "modified": 1443645832.748, + "optional": "(data/img/zero.*|data/optional.txt)", "sign": [ - 30041653970398729892154852118727733790145614202537425646336077462070348808967, - 96823925597554846684463773054016176426938620086211253074026312396122955360853 - ], - "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", + 33155653220731268227776289017011639520872180216646876377169089096034035969487, + 36744504416132878244552522451563313660303086381031784548929582417244124447603 + ], + "signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=", "signs": { - "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HKBBvaOi1v20ZuSVORtD4bBRRgf/85QDVy4HaaPX3fFDAKYmvUWK+Jbp3yIGElMmPoO2+YljFLyromAoEwWd6Eg=" - }, - "signs_required": 1, - "title": "ZeroBlog", + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HP+qomOGy0hTFX1HjHv8iQIL6E22qNynb+IijEblL2lm8SgsyiOxKGaVkD6/eE6xYGeYHSnhSii2Gw/04z3okNM=" + }, + "signs_required": 1, + "title": "ZeroBlog", "zeronet_version": "0.3.2" } \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/optional.txt b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/optional.txt new file mode 100644 index 00000000..3462721f --- /dev/null +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/optional.txt @@ -0,0 +1 @@ +hello! \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json index 9c0812d0..3bb3929f 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json @@ -1,15 +1,22 @@ { - "cert_auth_type": "web", - "cert_sign": "HBsTrjTmv+zD1iY93tSci8n9DqdEtYwzxJmRppn4/b+RYktcANGm5tXPOb+Duw3AJcgWDcGUvQVgN1D9QAwIlCw=", - "cert_user_id": "toruser@zeroid.bit", - "files": { - "data.json": { - "sha512": "4868b5e6d70a55d137db71c2e276bda80437e0235ac670962acc238071296b45", - "size": 168 - } - }, - "modified": 1432491109.11, - "signs": { - "1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9": "HMy7ZwwqE0Sk8O+5hTx/ejFW6KtIDbID6fGblCodUTpz4mJZ5GwApBHSVLMYL43vvGT/vKZOiQoJ5tQTeFVbbkk=" + "cert_auth_type": "web", + "cert_sign": "HBsTrjTmv+zD1iY93tSci8n9DqdEtYwzxJmRppn4/b+RYktcANGm5tXPOb+Duw3AJcgWDcGUvQVgN1D9QAwIlCw=", + "cert_user_id": "toruser@zeroid.bit", + "files": { + "data.json": { + "sha512": "4868b5e6d70a55d137db71c2e276bda80437e0235ac670962acc238071296b45", + "size": 168 } + }, + "files_optional": { + "peanut-butter-jelly-time.gif": { + "sha512": "a238fd27bda2a06f07f9f246954b34dcf82e6472aebdecc2c5dc1f01a50721ef", + "size": 1606 + } + }, + "modified": 1443645834.763, + "optional": ".*\\.(jpg|png|gif)", + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HD+/5Jmew6BotfeWsfpTFKXUVIY5MyjKQx5KRnT6WO0nMBLxaI6/sTb+6ZXq0tXjXNkmlt36/UICYQcYQjCRhkY=" + } } \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif new file mode 100644 index 00000000..54c69d1c Binary files /dev/null and b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif differ diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json index 9c0812d0..ada2c7ac 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json @@ -1,15 +1,15 @@ { - "cert_auth_type": "web", - "cert_sign": "HBsTrjTmv+zD1iY93tSci8n9DqdEtYwzxJmRppn4/b+RYktcANGm5tXPOb+Duw3AJcgWDcGUvQVgN1D9QAwIlCw=", - "cert_user_id": "toruser@zeroid.bit", - "files": { - "data.json": { - "sha512": "4868b5e6d70a55d137db71c2e276bda80437e0235ac670962acc238071296b45", - "size": 168 - } - }, - "modified": 1432491109.11, - "signs": { - "1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9": "HMy7ZwwqE0Sk8O+5hTx/ejFW6KtIDbID6fGblCodUTpz4mJZ5GwApBHSVLMYL43vvGT/vKZOiQoJ5tQTeFVbbkk=" + "cert_auth_type": "web", + "cert_sign": "HBsTrjTmv+zD1iY93tSci8n9DqdEtYwzxJmRppn4/b+RYktcANGm5tXPOb+Duw3AJcgWDcGUvQVgN1D9QAwIlCw=", + "cert_user_id": "toruser@zeroid.bit", + "files": { + "data.json": { + "sha512": "4868b5e6d70a55d137db71c2e276bda80437e0235ac670962acc238071296b45", + "size": 168 } + }, + "modified": 1443645835.157, + "signs": { + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HNIXDWV1kpqKtsJ+yrNKLvks/FDYIpVmx7xgkXPJ6NZiajCMHrgEwLH9QRiq6rs3nOCs0P08eRhlgZLvC+3U6ps=" + } } \ No newline at end of file diff --git a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json index dabbe5f8..b3894239 100644 --- a/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json +++ b/src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json @@ -1,9 +1,9 @@ { "files": {}, "ignore": ".*", - "modified": 1443088330.941, + "modified": 1443645833.247, "signs": { - "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G/YCfchtojDA7EjXk5Xa6af5EaEME14LDAvVE9P8PCDb2ncWN79ZTMsczAx7N3HYyM9Vdqn+8or4hh28z4ITKqU=" + "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G2EcaEKdzzfbpRITDcaBajNwjaIIJW3zp1YQGIMJcxfw3tLnn6uv/goImvbzvuTXKkl5fQKmBowK2Bg1xXJ3078=" }, "user_contents": { "cert_signers": { @@ -12,7 +12,9 @@ "permission_rules": { ".*": { "files_allowed": "data.json", + "files_allowed_optional": ".*\\.(png|jpg|gif)", "max_size": 10000, + "max_size_optional": 10000000, "signers": [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" ] }, "bitid/.*@zeroid.bit": { "max_size": 40000 }, diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py index 06d438e6..f4ee0e51 100644 --- a/src/Worker/WorkerManager.py +++ b/src/Worker/WorkerManager.py @@ -83,7 +83,9 @@ class WorkerManager: elif inner_path.endswith(".css"): priority += 5 # boost css files priority elif inner_path.endswith(".js"): - priority += 3 # boost js files priority + priority += 4 # boost js files priority + elif inner_path.endswith("dbschema.json"): + priority += 3 # boost database specification elif inner_path.endswith("content.json"): priority += 1 # boost included content.json files priority a bit elif inner_path.endswith(".json"): diff --git a/src/util/helper.py b/src/util/helper.py index 2d8c4f33..b78d06ef 100644 --- a/src/util/helper.py +++ b/src/util/helper.py @@ -1,6 +1,7 @@ import os import socket import struct +import re def atomicWrite(dest, content, mode="w"): @@ -16,10 +17,27 @@ def shellquote(*args): else: return tuple(['"%s"' % arg.replace('"', "") for arg in args]) + # ip, port to packed 6byte format def packAddress(ip, port): return socket.inet_aton(ip) + struct.pack("H", port) + # From 6byte format to ip, port def unpackAddress(packed): - return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0] \ No newline at end of file + return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0] + + +# Get dir from file +# Return: data/site/content.json -> data/site +def getDirname(path): + file_dir = re.sub("[^/]*?$", "", path).rstrip("/") + if file_dir: + file_dir += "/" # Add / at end if its not the root + return file_dir + + +# Get dir from file +# Return: data/site/content.json -> content.json +def getFilename(path): + return re.sub("^.*/", "", path)