oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers

Browse source
This commit is contained in:
HelloZeroNet 2015-10-01 01:35:13 +02:00
parent a7d8d488da
commit 9d7d4f1552
22 changed files with 486 additions and 220 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
requirements.txt
View file

@ -1,2 +1,2 @@
gevent==1.0.1
msgpack-python==0.4.4
gevent>=1.0.1
msgpack-python>=0.4.4

2
src/Config.py
View file

@ -8,7 +8,7 @@ class Config(object):
def __init__(self, argv):
self.version = "0.3.2"
self.rev = 465
self.rev = 467
self.argv = argv
self.action = None
self.createParser()

81
src/Content/ContentManager.py
View file

@ -9,7 +9,7 @@ import gevent
from Debug import Debug
from Crypt import CryptHash
from Config import config
from util import helper
class ContentManager(object):
@ -26,8 +26,8 @@ class ContentManager(object):
content_inner_path = content_inner_path.strip("/") # Remove / from begning
old_content = self.contents.get(content_inner_path)
content_path = self.site.storage.getPath(content_inner_path)
content_dir = self.toDir(self.site.storage.getPath(content_inner_path))
content_inner_dir = self.toDir(content_inner_path)
content_dir = helper.getDirname(self.site.storage.getPath(content_inner_path))
content_inner_dir = helper.getDirname(content_inner_path)
if os.path.isfile(content_path):
try:
@ -140,16 +140,29 @@ class ContentManager(object):
while True:
content_inner_path = "%s/content.json" % "/".join(dirs)
content = self.contents.get(content_inner_path.strip("/"))
if content and "files" in content: # Check if content.json exists
# Check in files
if content and "files" in content:
back = content["files"].get("/".join(inner_path_parts))
if back:
back["content_inner_path"] = content_inner_path
back["optional"] = False
return back
if content and "user_contents" in content: # User dir
# Check in optional files
if content and "files_optional" in content: # Check if file in this content.json
back = content["files_optional"].get("/".join(inner_path_parts))
if back:
back["content_inner_path"] = content_inner_path
back["optional"] = True
return back
# Return the rules if user dir
if content and "user_contents" in content:
back = content["user_contents"]
# Content.json is in the users dir
back["content_inner_path"] = re.sub("(.*)/.*?$", "\\1/content.json", inner_path)
back["optional"] = None
return back
# No inner path in this dir, lets try the parent dir
@ -234,6 +247,7 @@ class ContentManager(object):
rules["signers"] = []
rules["signers"].append(user_address) # Add user as valid signer
rules["user_address"] = user_address
rules["includes_allowed"] = False
return rules
@ -243,7 +257,7 @@ class ContentManager(object):
files_optional_node = {}
for file_relative_path in self.site.storage.list(dir_inner_path):
file_name = self.toFilename(file_relative_path)
file_name = helper.getFilename(file_relative_path)
ignored = optional = False
if file_name == "content.json":
@ -283,12 +297,12 @@ class ContentManager(object):
if extend:
content.update(extend) # Add custom fields
directory = self.toDir(self.site.storage.getPath(inner_path))
inner_directory = self.toDir(inner_path)
directory = helper.getDirname(self.site.storage.getPath(inner_path))
inner_directory = helper.getDirname(inner_path)
self.log.info("Opening site data directory: %s..." % directory)
changed_files = [inner_path]
files_node, files_optional_node = self.hashFiles(self.toDir(inner_path), content.get("ignore"), content.get("optional"))
files_node, files_optional_node = self.hashFiles(helper.getDirname(inner_path), content.get("ignore"), content.get("optional"))
# Find changed files
files_merged = files_node.copy()
@ -310,13 +324,17 @@ class ContentManager(object):
new_content = content.copy() # Create a copy of current content.json
new_content["files"] = files_node # Add files sha512 hash
if files_optional_node:
new_content["files_optional_node"] = files_optional_node
new_content["files_optional"] = files_optional_node
elif "files_optional" in new_content:
del new_content["files_optional"]
new_content["modified"] = time.time() # Add timestamp
if inner_path == "content.json":
new_content["address"] = self.site.address
new_content["zeronet_version"] = config.version
new_content["signs_required"] = content.get("signs_required", 1)
# Verify private key
from Crypt import CryptBitcoin
self.log.info("Verifying private key...")
privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey)
@ -409,6 +427,7 @@ class ContentManager(object):
# Return: True or False
def verifyContent(self, inner_path, content):
content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content
content_size_optional = sum([file["size"] for file in content.get("files_optional", {}).values()])
site_size = self.getTotalSize(ignore=inner_path) + content_size # Site size without old content
if site_size > self.site.settings.get("size", 0):
self.site.settings["size"] = site_size # Save to settings if larger
@ -433,23 +452,34 @@ class ContentManager(object):
return False
# Check include size limit
if rules.get("max_size"): # Include size limit
if rules.get("max_size") is not None: # Include size limit
if content_size > rules["max_size"]:
self.log.error("%s: Include too large %s > %s" % (inner_path, content_size, rules["max_size"]))
return False
if rules.get("max_size_optional") is not None: # Include optional files limit
if content_size_optional > rules["max_size_optional"]:
self.log.error("%s: Include optional files too large %s > %s" % (inner_path, content_size_optional, rules["max_size_optional"]))
return False
# Filename limit
if rules.get("files_allowed"):
for file_inner_path in content["files"].keys():
if not re.match("^%s$" % rules["files_allowed"], file_inner_path):
self.log.error("%s %s: File not allowed" % (inner_path, file_inner_path))
return False
if rules.get("files_allowed_optional"):
for file_inner_path in content.get("files_optional", {}).keys():
if not re.match("^%s$" % rules["files_allowed_optional"], file_inner_path):
self.log.error("%s %s: Optional file not allowed" % (inner_path, file_inner_path))
return False
# Check if content includes allowed
if rules.get("includes_allowed") is False and content.get("includes"):
self.log.error("%s: Includes not allowed" % inner_path)
return False # Includes not allowed
# Filename limit
if rules.get("files_allowed"):
for file_inner_path in content["files"].keys():
if not re.match("^%s$" % rules["files_allowed"], file_inner_path):
self.log.error("%s: File not allowed" % file_inner_path)
return False
return True # All good
# Verify file validity
@ -507,7 +537,7 @@ class ContentManager(object):
valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
if valid_signs >= signs_required:
break # Break if we has enough signs
self.log.debug("%s: Valid signs: %s/%s" % (inner_path, valid_signs, signs_required))
return valid_signs >= signs_required
else: # Old style signing
return CryptBitcoin.verify(sign_content, self.site.address, sign)
@ -537,19 +567,6 @@ class ContentManager(object):
self.log.error("File not in content.json: %s" % inner_path)
return False
# Get dir from file
# Return: data/site/content.json -> data/site
def toDir(self, inner_path):
file_dir = re.sub("[^/]*?$", "", inner_path).strip("/")
if file_dir:
file_dir += "/" # Add / at end if its not the root
return file_dir
# Get dir from file
# Return: data/site/content.json -> data/site
def toFilename(self, inner_path):
return re.sub("^.*/", "", inner_path)
if __name__ == "__main__":
def testSign():

38
src/Peer/Peer.py
View file

@ -1,6 +1,8 @@
import logging
import gevent
import time
import array
import gevent
from cStringIO import StringIO
from Debug import Debug
@ -14,8 +16,8 @@ if config.use_tempfiles:
# Communicate remote peers
class Peer(object):
__slots__ = (
"ip", "port", "site", "key", "connection", "last_found", "last_response",
"last_ping", "added", "connection_error", "hash_failed", "download_bytes", "download_time"
"ip", "port", "site", "key", "connection", "last_found", "last_response", "last_ping", "last_hashfield",
"hashfield", "added", "connection_error", "hash_failed", "download_bytes", "download_time"
)
def __init__(self, ip, port, site=None):
@ -25,6 +27,8 @@ class Peer(object):
self.key = "%s:%s" % (ip, port)
self.connection = None
self.hashfield = array.array("H") # Got optional files hash_id
self.last_hashfield = None # Last time hashfiled downloaded
self.last_found = time.time() # Time of last found in the torrent tracker
self.last_response = None # Time of last successful response from peer
self.last_ping = None # Last response time for ping
@ -230,6 +234,34 @@ class Peer(object):
if self.connection:
self.connection.close()
# - HASHFIELD -
def updateHashfield(self, force=False):
# Don't update hashfield again in 15 min
if self.last_hashfield and time.time() - self.last_hashfield > 60 * 15 and not force:
return False
response = self.request("getHashfield", {"site": self.site.address})
if not response or "error" in response:
return False
self.last_hashfield = time.time()
self.hashfield = response["hashfield"]
return self.hashfield
def setHashfield(self, hashfield_dump):
self.hashfield.fromstring(hashfield_dump)
def hasHash(self, hash_id):
return hash_id in self.hashfield
# Return: ["ip:port", "ip:port",...]
def findHash(self, hash_id):
response = self.request("findHash", {"site": self.site.address, "hash_id": hash_id})
if not response or "error" in response:
return False
return [helper.unpackAddress(peer) for peer in response["peers"]]
# - EVENTS -
# On connection error

4
src/Site/Site.py
View file

@ -112,7 +112,7 @@ class Site:
s = time.time()
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
content_inner_dir = self.content_manager.toDir(inner_path)
content_inner_dir = helper.getDirname(inner_path)
if not found:
self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications))
if check_modifications: # Download failed, but check modifications if its succed later
@ -386,7 +386,7 @@ class Site:
# Copy files
for content_inner_path, content in self.content_manager.contents.items():
for file_relative_path in sorted(content["files"].keys()):
file_inner_path = self.content_manager.toDir(content_inner_path) + file_relative_path # Relative to content.json
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to content.json
file_inner_path = file_inner_path.strip("/") # Strip leading /
if file_inner_path.split("/")[0] in default_dirs: # Dont copy directories that has -default postfixed alternative
self.log.debug("[SKIP] %s (has default alternative)" % file_inner_path)

16
src/Site/SiteStorage.py
View file

@ -3,7 +3,6 @@ import re
import shutil
import json
import time
import sys
import sqlite3
import gevent.event
@ -11,6 +10,7 @@ import gevent.event
from Db import Db
from Debug import Debug
from Config import config
from util import helper
class SiteStorage:
@ -98,7 +98,7 @@ class SiteStorage:
for file_relative_path in content["files"].keys():
if not file_relative_path.endswith(".json"):
continue # We only interesed in json files
content_inner_path_dir = self.site.content_manager.toDir(content_inner_path) # Content.json dir relative to site
content_inner_path_dir = helper.getDirname(content_inner_path) # Content.json dir relative to site
file_inner_path = content_inner_path_dir + file_relative_path # File Relative to site dir
file_inner_path = file_inner_path.strip("/") # Strip leading /
file_path = self.getPath(file_inner_path)
@ -170,7 +170,6 @@ class SiteStorage:
else:
yield file_name
# Site content updated
def onUpdated(self, inner_path):
file_path = self.getPath(inner_path)
@ -255,7 +254,7 @@ class SiteStorage:
self.log.debug("[MISSING] %s" % content_inner_path)
bad_files.append(content_inner_path)
for file_relative_path in content["files"].keys():
file_inner_path = self.site.content_manager.toDir(content_inner_path) + file_relative_path # Relative to site dir
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
file_inner_path = file_inner_path.strip("/") # Strip leading /
file_path = self.getPath(file_inner_path)
if not os.path.isfile(file_path):
@ -304,8 +303,13 @@ class SiteStorage:
files = [] # Get filenames
for content_inner_path, content in self.site.content_manager.contents.items():
files.append(content_inner_path)
for file_relative_path in content["files"].keys():
file_inner_path = self.site.content_manager.toDir(content_inner_path) + file_relative_path # Relative to site dir
# Add normal files
for file_relative_path in content.get("files", {}).keys():
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
files.append(file_inner_path)
# Add optional files
for file_relative_path in content.get("files_optional", {}).keys():
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
files.append(file_inner_path)
for inner_path in files:

17
src/Test/Spy.py Normal file
View file

@ -0,0 +1,17 @@
class Spy:
def __init__(self, obj, func_name):
self.obj = obj
self.func_name = func_name
self.func_original = getattr(self.obj, func_name)
self.calls = []
def __enter__(self, *args, **kwargs):
def loggedFunc(cls, *args, **kwags):
print "Logging", self, args, kwargs
self.calls.append(args)
return self.func_original(cls, *args, **kwargs)
setattr(self.obj, self.func_name, loggedFunc)
return self.calls
def __exit__(self, *args, **kwargs):
setattr(self.obj, self.func_name, self.func_original)

56
src/Test/TestContent.py
View file

@ -9,7 +9,7 @@ from Crypt import CryptBitcoin
@pytest.mark.usefixtures("resetSettings")
class TestContent:
def testIncludes(self, site):
def testInclude(self, site):
# Rules defined in parent content.json
rules = site.content_manager.getRules("data/test_include/content.json")
@ -34,7 +34,7 @@ class TestContent:
# Valid signers for root content.json
assert site.content_manager.getValidSigners("content.json") == ["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"]
def testLimits(self, site):
def testInlcudeLimits(self, site):
privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv"
# Data validation
data_dict = {
@ -76,3 +76,55 @@ class TestContent:
data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)}
data = StringIO(json.dumps(data_dict))
assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False)
@pytest.mark.parametrize("inner_path", ["content.json", "data/test_include/content.json", "data/users/content.json"])
def testSign(self, site, inner_path):
# Bad privatekey
assert not site.content_manager.sign(inner_path, privatekey="5aaa3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMnaa", filewrite=False)
# Good privatekey
content = site.content_manager.sign(inner_path, privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
content_old = site.content_manager.contents[inner_path] # Content before the sign
assert not content_old == content # Timestamp changed
assert site.address in content["signs"] # Used the site's private key to sign
if inner_path == "content.json":
assert len(content["files"]) == 17
elif inner_path == "data/test-include/content.json":
assert len(content["files"]) == 1
elif inner_path == "data/users/content.json":
assert len(content["files"]) == 0
# Everything should be same as before except the modified timestamp and the signs
assert (
{key: val for key, val in content_old.items() if key not in ["modified", "signs", "sign"]}
==
{key: val for key, val in content.items() if key not in ["modified", "signs", "sign"]}
)
def testSignOptionalFiles(self, site):
site.content_manager.contents["content.json"]["optional"] = "((data/img/zero.*))"
content_optional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
del site.content_manager.contents["content.json"]["optional"]
content_nooptional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
assert len(content_nooptional.get("files_optional", {})) == 0
assert len(content_optional["files_optional"]) > 0
assert len(content_nooptional["files"]) > len(content_optional["files"])
def testFileInfo(self, site):
assert "sha512" in site.content_manager.getFileInfo("index.html")
assert not site.content_manager.getFileInfo("notexist")
# Optional file
file_info_optional = site.content_manager.getFileInfo("data/optional.txt")
assert "sha512" in file_info_optional
assert file_info_optional["optional"] is True
# Not exists yet user content.json
assert "cert_signers" in site.content_manager.getFileInfo("data/users/unknown/content.json")
# Optional user file
file_info_optional = site.content_manager.getFileInfo("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif")
assert "sha512" in file_info_optional
assert file_info_optional["optional"] is True

60
src/Test/TestContentUser.py
View file

@ -50,6 +50,66 @@ class TestUserContent:
rules = site.content_manager.getRules("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content)
assert rules is False
def testVerify(self, site):
privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT
user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json"
data_dict = site.content_manager.contents[user_inner_path]
users_content = site.content_manager.contents["data/users/content.json"]
data = StringIO(json.dumps(data_dict))
assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
# Test max size exception by setting allowed to 0
rules = site.content_manager.getRules(user_inner_path, data_dict)
assert rules["max_size"] == 10000
assert users_content["user_contents"]["permission_rules"][".*"]["max_size"] == 10000
users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 0
rules = site.content_manager.getRules(user_inner_path, data_dict)
assert rules["max_size"] == 0
data = StringIO(json.dumps(data_dict))
assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 10000 # Reset
# Test max optional size exception
# 1 MB gif = Allowed
data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024
del data_dict["signs"] # Remove signs before signing
data_dict["signs"] = {
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
}
data = StringIO(json.dumps(data_dict))
assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
# 100 MB gif = Not allowed
data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 100 * 1024 * 1024
del data_dict["signs"] # Remove signs before signing
data_dict["signs"] = {
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
}
data = StringIO(json.dumps(data_dict))
assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024 # Reset
# hello.exe = Not allowed
data_dict["files_optional"]["hello.exe"] = data_dict["files_optional"]["peanut-butter-jelly-time.gif"]
del data_dict["signs"] # Remove signs before signing
data_dict["signs"] = {
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
}
data = StringIO(json.dumps(data_dict))
assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
del data_dict["files_optional"]["hello.exe"] # Reset
# Includes not allowed in user content
data_dict["includes"] = { "other.json": { } }
del data_dict["signs"] # Remove signs before signing
data_dict["signs"] = {
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
}
data = StringIO(json.dumps(data_dict))
assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
def testCert(self, site):
# user_addr = "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C"
user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A"

52
src/Test/TestHelper.py
View file

@ -1,17 +1,39 @@
class Spy:
def __init__(self, obj, func_name):
self.obj = obj
self.func_name = func_name
self.func_original = getattr(self.obj, func_name)
self.calls = []
import socket
def __enter__(self, *args, **kwargs):
def loggedFunc(cls, *args, **kwags):
print "Logging", self, args, kwargs
self.calls.append(args)
return self.func_original(cls, *args, **kwargs)
setattr(self.obj, self.func_name, loggedFunc)
return self.calls
import pytest
from util import helper
def __exit__(self, *args, **kwargs):
setattr(self.obj, self.func_name, self.func_original)
@pytest.mark.usefixtures("resetSettings")
class TestHelper:
def testShellquote(self):
assert helper.shellquote("hel'lo") == "\"hel'lo\"" # Allow '
assert helper.shellquote('hel"lo') == '"hello"' # Remove "
assert helper.shellquote("hel'lo", 'hel"lo') == ('"hel\'lo"', '"hello"')
def testPackAddress(self):
assert len(helper.packAddress("1.1.1.1", 1)) == 6
assert helper.unpackAddress(helper.packAddress("1.1.1.1", 1)) == ("1.1.1.1", 1)
with pytest.raises(socket.error):
helper.packAddress("999.1.1.1", 1)
with pytest.raises(socket.error):
helper.unpackAddress("X")
def testGetDirname(self):
assert helper.getDirname("data/users/content.json") == "data/users/"
assert helper.getDirname("data/users") == "data/"
assert helper.getDirname("") == ""
assert helper.getDirname("content.json") == ""
assert helper.getDirname("data/users/") == "data/users/"
assert helper.getDirname("/data/users/content.json") == "/data/users/"
def testGetFilename(self):
assert helper.getFilename("data/users/content.json") == "content.json"
assert helper.getFilename("data/users") == "users"
assert helper.getFilename("") == ""
assert helper.getFilename("content.json") == "content.json"
assert helper.getFilename("data/users/") == ""
assert helper.getFilename("/data/users/content.json") == "content.json"

33
src/Test/TestSite.py
View file

@ -62,36 +62,3 @@ class TestSite:
assert new_site.address in SiteManager.site_manager.sites
SiteManager.site_manager.delete(new_site.address)
assert new_site.address not in SiteManager.site_manager.sites
@pytest.mark.parametrize("inner_path", ["content.json", "data/test_include/content.json", "data/users/content.json"])
def testSign(self, site, inner_path):
# Bad privatekey
assert not site.content_manager.sign(inner_path, privatekey="5aaa3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMnaa", filewrite=False)
# Good privatekey
content = site.content_manager.sign(inner_path, privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
content_old = site.content_manager.contents[inner_path] # Content before the sign
assert not content_old == content # Timestamp changed
assert site.address in content["signs"] # Used the site's private key to sign
if inner_path == "content.json":
assert len(content["files"]) == 24
elif inner_path == "data/test-include/content.json":
assert len(content["files"]) == 1
elif inner_path == "data/users/content.json":
assert len(content["files"]) == 0
# Everything should be same as before except the modified timestamp and the signs
assert (
{key: val for key, val in content_old.items() if key not in ["modified", "signs", "sign"]}
==
{key: val for key, val in content.items() if key not in ["modified", "signs", "sign"]}
)
def testSignOptionalFiles(self, site):
site.content_manager.contents["content.json"]["optional"] = "((data/img/zero.*))"
content_optional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
del site.content_manager.contents["content.json"]["optional"]
content_nooptional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
assert len(content_nooptional["files"]) > len(content_optional["files"])

63
src/Test/TestSiteDownload.py Normal file
View file

@ -0,0 +1,63 @@
import pytest
import mock
import time
from Connection import ConnectionServer
from Config import config
from Site import Site
from File import FileRequest
import Spy
@pytest.mark.usefixtures("resetTempSettings")
@pytest.mark.usefixtures("resetSettings")
class TestWorker:
def testDownload(self, file_server, site, site_temp):
client = ConnectionServer("127.0.0.1", 1545)
assert site.storage.directory == config.data_dir + "/" + site.address
assert site_temp.storage.directory == config.data_dir + "-temp/" + site.address
# Init source server
site.connection_server = file_server
file_server.sites[site.address] = site
# Init client server
site_temp.connection_server = client
site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net
site_temp.addPeer("127.0.0.1", 1544)
with Spy.Spy(FileRequest, "route") as requests:
def boostRequest(inner_path):
# I really want these file
if inner_path == "index.html":
print "needFile"
site_temp.needFile("data/img/multiuser.png", priority=9, blocking=False)
site_temp.needFile("data/img/direct_domains.png", priority=10, blocking=False)
site_temp.onFileDone.append(boostRequest)
site_temp.download(blind_includes=True).join(timeout=5)
file_requests = [request[2]["inner_path"] for request in requests if request[0] in ("getFile", "streamFile")]
# Test priority
assert file_requests[0:2] == ["content.json", "index.html"] # Must-have files
assert file_requests[2:4] == ["data/img/direct_domains.png", "data/img/multiuser.png"] # Directly requested files
assert file_requests[4:6] == ["css/all.css", "js/all.js"] # Important assets
assert file_requests[6] == "dbschema.json" # Database map
assert "-default" in file_requests[-1] # Put default files for cloning to the end
# Check files
bad_files = site_temp.storage.verifyFiles(quick_check=True)
# -1 because data/users/1J6... user has invalid cert
assert len(site_temp.content_manager.contents) == len(site.content_manager.contents) - 1
assert not bad_files
# Optional file
assert not site_temp.storage.isFile("data/optional.txt")
assert site.storage.isFile("data/optional.txt")
site_temp.needFile("data/optional.txt")
assert site_temp.storage.isFile("data/optional.txt")
# Optional user file
assert not site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif")
site_temp.needFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif")
assert site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif")
assert site_temp.storage.deleteFiles()

6
src/Test/TestSiteStorage.py
View file

@ -1,15 +1,13 @@
import shutil
import os
import pytest
from Site import SiteManager
@pytest.mark.usefixtures("resetSettings")
class TestSiteStorage:
def testList(self, site):
# Rootdir
list_root = list(site.storage.list(""))
assert "content.json" in list_root
assert "css/all.css" in list_root
# Subdir
assert list(site.storage.list("data-default")) == ["data.json", "users/content-default.json"]

3
src/Test/conftest.py
View file

@ -22,6 +22,7 @@ from Config import config
config.argv = ["none"] # Dont pass any argv to config parser
config.parse()
config.data_dir = "src/Test/testdata" # Use test data for unittests
config.debug_socket = True # Use test data for unittests
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
from Site import Site
@ -76,7 +77,7 @@ def resetTempSettings(request):
request.addfinalizer(cleanup)
@pytest.fixture(scope="session")
@pytest.fixture()
def site():
site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
return site

51
src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/content.json vendored
View file

@ -56,6 +56,24 @@
"sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f",
"size": 19040
},
"dbschema.json": {
"sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8",
"size": 1529
},
"img/loading.gif": {
"sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00",
"size": 723
},
"index.html": {
"sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3",
"size": 4667
},
"js/all.js": {
"sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c",
"size": 201178
}
},
"files_optional": {
"data/img/zeroblog-comments.png": {
"sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4",
"size": 24001
@ -80,28 +98,12 @@
"sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8",
"size": 26606
},
"data/test_include/data.json": {
"sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906",
"size": 505
},
"dbschema.json": {
"sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8",
"size": 1529
},
"img/loading.gif": {
"sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00",
"size": 723
},
"index.html": {
"sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3",
"size": 4667
},
"js/all.js": {
"sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c",
"size": 201178
"data/optional.txt": {
"sha512": "c6f81db0e9f8206c971c9e5826e3ba823ffbb1a3a900f8047652a8bf78ea98fd",
"size": 6
}
},
"ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)",
"ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*|data/test_include/.*)",
"includes": {
"data/test_include/content.json": {
"added": 1424976057,
@ -118,14 +120,15 @@
"signers_required": 1
}
},
"modified": 1443393859.801,
"modified": 1443645832.748,
"optional": "(data/img/zero.*|data/optional.txt)",
"sign": [
30041653970398729892154852118727733790145614202537425646336077462070348808967,
96823925597554846684463773054016176426938620086211253074026312396122955360853
33155653220731268227776289017011639520872180216646876377169089096034035969487,
36744504416132878244552522451563313660303086381031784548929582417244124447603
],
"signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=",
"signs": {
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HKBBvaOi1v20ZuSVORtD4bBRRgf/85QDVy4HaaPX3fFDAKYmvUWK+Jbp3yIGElMmPoO2+YljFLyromAoEwWd6Eg="
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HP+qomOGy0hTFX1HjHv8iQIL6E22qNynb+IijEblL2lm8SgsyiOxKGaVkD6/eE6xYGeYHSnhSii2Gw/04z3okNM="
},
"signs_required": 1,
"title": "ZeroBlog",

1
src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/optional.txt vendored Normal file
View file

@ -0,0 +1 @@
hello!

11
src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json vendored
View file

@ -8,8 +8,15 @@
"size": 168
}
},
"modified": 1432491109.11,
"files_optional": {
"peanut-butter-jelly-time.gif": {
"sha512": "a238fd27bda2a06f07f9f246954b34dcf82e6472aebdecc2c5dc1f01a50721ef",
"size": 1606
}
},
"modified": 1443645834.763,
"optional": ".*\\.(jpg|png|gif)",
"signs": {
"1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9": "HMy7ZwwqE0Sk8O+5hTx/ejFW6KtIDbID6fGblCodUTpz4mJZ5GwApBHSVLMYL43vvGT/vKZOiQoJ5tQTeFVbbkk="
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HD+/5Jmew6BotfeWsfpTFKXUVIY5MyjKQx5KRnT6WO0nMBLxaI6/sTb+6ZXq0tXjXNkmlt36/UICYQcYQjCRhkY="
}
}

BIN
src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.6 KiB

4
src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json vendored
View file

@ -8,8 +8,8 @@
"size": 168
}
},
"modified": 1432491109.11,
"modified": 1443645835.157,
"signs": {
"1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9": "HMy7ZwwqE0Sk8O+5hTx/ejFW6KtIDbID6fGblCodUTpz4mJZ5GwApBHSVLMYL43vvGT/vKZOiQoJ5tQTeFVbbkk="
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HNIXDWV1kpqKtsJ+yrNKLvks/FDYIpVmx7xgkXPJ6NZiajCMHrgEwLH9QRiq6rs3nOCs0P08eRhlgZLvC+3U6ps="
}
}

6
src/Test/testdata/1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT/data/users/content.json vendored
View file

@ -1,9 +1,9 @@
{
"files": {},
"ignore": ".*",
"modified": 1443088330.941,
"modified": 1443645833.247,
"signs": {
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G/YCfchtojDA7EjXk5Xa6af5EaEME14LDAvVE9P8PCDb2ncWN79ZTMsczAx7N3HYyM9Vdqn+8or4hh28z4ITKqU="
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "G2EcaEKdzzfbpRITDcaBajNwjaIIJW3zp1YQGIMJcxfw3tLnn6uv/goImvbzvuTXKkl5fQKmBowK2Bg1xXJ3078="
},
"user_contents": {
"cert_signers": {
@ -12,7 +12,9 @@
"permission_rules": {
".*": {
"files_allowed": "data.json",
"files_allowed_optional": ".*\\.(png|jpg|gif)",
"max_size": 10000,
"max_size_optional": 10000000,
"signers": [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" ]
},
"bitid/.*@zeroid.bit": { "max_size": 40000 },

4
src/Worker/WorkerManager.py
View file

@ -83,7 +83,9 @@ class WorkerManager:
elif inner_path.endswith(".css"):
priority += 5 # boost css files priority
elif inner_path.endswith(".js"):
priority += 3 # boost js files priority
priority += 4 # boost js files priority
elif inner_path.endswith("dbschema.json"):
priority += 3 # boost database specification
elif inner_path.endswith("content.json"):
priority += 1 # boost included content.json files priority a bit
elif inner_path.endswith(".json"):

18
src/util/helper.py
View file

@ -1,6 +1,7 @@
import os
import socket
import struct
import re
def atomicWrite(dest, content, mode="w"):
@ -16,10 +17,27 @@ def shellquote(*args):
else:
return tuple(['"%s"' % arg.replace('"', "") for arg in args])
# ip, port to packed 6byte format
def packAddress(ip, port):
return socket.inet_aton(ip) + struct.pack("H", port)
# From 6byte format to ip, port
def unpackAddress(packed):
return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0]
# Get dir from file
# Return: data/site/content.json -> data/site
def getDirname(path):
file_dir = re.sub("[^/]*?$", "", path).rstrip("/")
if file_dir:
file_dir += "/" # Add / at end if its not the root
return file_dir
# Get dir from file
# Return: data/site/content.json -> content.json
def getFilename(path):
return re.sub("^.*/", "", path)