Rev465, Display memory dump only in debug mode, Use sys.exit in trayicon, Optional files hashing, List function in SiteStorage, Test signing content, Test sign optional files, Test site storage

This commit is contained in:
HelloZeroNet 2015-09-28 22:07:26 +02:00
parent 39413b9755
commit a7d8d488da
9 changed files with 216 additions and 107 deletions

View file

@ -147,8 +147,13 @@ class UiRequestPlugin(object):
yield "<br></td></tr>"
yield "</table>"
# No more if not in debug mode
if not config.debug:
raise StopIteration
# Object types
obj_count = {}
for obj in gc.get_objects():
obj_type = str(type(obj))
@ -250,10 +255,17 @@ class UiRequestPlugin(object):
yield "Done in %.1f" % (time.time() - s)
def actionDumpobj(self):
import gc
import sys
self.sendHeader()
# No more if not in debug mode
if not config.debug:
yield "Not in debug mode"
raise StopIteration
class_filter = self.get.get("class")
yield """
@ -276,10 +288,17 @@ class UiRequestPlugin(object):
gc.collect() # Implicit grabage collection
def actionListobj(self):
import gc
import sys
self.sendHeader()
# No more if not in debug mode
if not config.debug:
yield "Not in debug mode"
raise StopIteration
type_filter = self.get.get("type")
yield """

View file

@ -63,9 +63,9 @@ class ActionsPlugin(object):
def quit(self):
self.icon.die()
time.sleep(0.1)
self.main.ui_server.stop()
self.main.file_server.stop()
# sys.exit()
sys.exit()
#self.main.ui_server.stop()
#self.main.file_server.stop()
def opensite(self, url):
import webbrowser

View file

@ -8,7 +8,7 @@ class Config(object):
def __init__(self, argv):
self.version = "0.3.2"
self.rev = 452
self.rev = 465
self.argv = argv
self.action = None
self.createParser()

View file

@ -237,6 +237,37 @@ class ContentManager(object):
return rules
# Hash files in directory
def hashFiles(self, dir_inner_path, ignore_pattern=None, optional_pattern=None):
files_node = {}
files_optional_node = {}
for file_relative_path in self.site.storage.list(dir_inner_path):
file_name = self.toFilename(file_relative_path)
ignored = optional = False
if file_name == "content.json":
ignored = True
elif ignore_pattern and re.match(ignore_pattern, file_relative_path):
ignored = True
elif file_name.startswith("."):
ignored = True
elif optional_pattern and re.match(optional_pattern, file_relative_path):
optional = True
if ignored: # Ignore content.json, definied regexp and files starting with .
self.log.info("- [SKIPPED] %s" % file_relative_path)
else:
file_path = self.site.storage.getPath(dir_inner_path + "/" + file_relative_path)
sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file
if optional:
self.log.info("- [OPTIONAL] %s (SHA512: %s)" % (file_relative_path, sha512sum))
files_optional_node[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
else:
self.log.info("- %s (SHA512: %s)" % (file_relative_path, sha512sum))
files_node[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
return files_node, files_optional_node
# Create and sign a content.json
# Return: The new content if filewrite = False
def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None):
@ -253,35 +284,20 @@ class ContentManager(object):
content.update(extend) # Add custom fields
directory = self.toDir(self.site.storage.getPath(inner_path))
inner_directory = self.toDir(inner_path)
self.log.info("Opening site data directory: %s..." % directory)
hashed_files = {}
changed_files = [inner_path]
for root, dirs, files in os.walk(directory):
for file_name in files:
file_path = self.site.storage.getPath("%s/%s" % (root.strip("/"), file_name))
file_inner_path = re.sub(re.escape(directory), "", file_path)
files_node, files_optional_node = self.hashFiles(self.toDir(inner_path), content.get("ignore"), content.get("optional"))
if file_name == "content.json":
ignored = True
elif content.get("ignore") and re.match(content["ignore"], file_inner_path):
ignored = True
elif file_name.startswith("."):
ignored = True
else:
ignored = False
if ignored: # Ignore content.json, definied regexp and files starting with .
self.log.info("- [SKIPPED] %s" % file_inner_path)
else:
sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file
self.log.info("- %s (SHA512: %s)" % (file_inner_path, sha512sum))
hashed_files[file_inner_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
if (
file_inner_path in content["files"].keys()
and hashed_files[file_inner_path]["sha512"] != content["files"][file_inner_path].get("sha512")
):
changed_files.append(file_path)
# Find changed files
files_merged = files_node.copy()
files_merged.update(files_optional_node)
for file_relative_path, file_details in files_merged.iteritems():
old_hash = content["files"].get(file_relative_path, {}).get("sha512")
new_hash = files_merged[file_relative_path]["sha512"]
if old_hash != new_hash:
changed_files.append(inner_directory + file_relative_path)
self.log.debug("Changed files: %s" % changed_files)
if update_changed_files:
@ -292,7 +308,9 @@ class ContentManager(object):
self.log.info("Adding timestamp and sha512sums to new content.json...")
new_content = content.copy() # Create a copy of current content.json
new_content["files"] = hashed_files # Add files sha512 hash
new_content["files"] = files_node # Add files sha512 hash
if files_optional_node:
new_content["files_optional_node"] = files_optional_node
new_content["modified"] = time.time() # Add timestamp
if inner_path == "content.json":
new_content["address"] = self.site.address
@ -336,7 +354,7 @@ class ContentManager(object):
oldsign_content = json.dumps(new_content, sort_keys=True)
new_content["sign"] = CryptBitcoin.signOld(oldsign_content, privatekey)
if not self.validContent(inner_path, new_content):
if not self.verifyContent(inner_path, new_content):
self.log.error("Sign failed: Invalid content")
return False
@ -389,7 +407,7 @@ class ContentManager(object):
# Checks if the content.json content is valid
# Return: True or False
def validContent(self, inner_path, content):
def verifyContent(self, inner_path, content):
content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content
site_size = self.getTotalSize(ignore=inner_path) + content_size # Site size without old content
if site_size > self.site.settings.get("size", 0):
@ -465,7 +483,7 @@ class ContentManager(object):
del(new_content["signs"]) # The file signed without the signs
sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace
if not self.validContent(inner_path, new_content):
if not self.verifyContent(inner_path, new_content):
return False # Content not valid (files too large, invalid files)
if signs: # New style signing
@ -527,6 +545,11 @@ class ContentManager(object):
file_dir += "/" # Add / at end if its not the root
return file_dir
# Get dir from file
# Return: data/site/content.json -> data/site
def toFilename(self, inner_path):
return re.sub("^.*/", "", inner_path)
if __name__ == "__main__":
def testSign():

View file

@ -158,6 +158,19 @@ class SiteStorage:
file_path = self.getPath(inner_path)
os.unlink(file_path)
# List files from a directory
def list(self, dir_inner_path):
directory = self.getPath(dir_inner_path)
for root, dirs, files in os.walk(directory):
root = root.replace("\\", "/")
root_relative_path = re.sub("^%s" % re.escape(directory), "", root).lstrip("/")
for file_name in files:
if root_relative_path: # Not root dir
yield root_relative_path + "/" + file_name
else:
yield file_name
# Site content updated
def onUpdated(self, inner_path):
file_path = self.getPath(inner_path)
@ -215,6 +228,9 @@ class SiteStorage:
inner_path = inner_path.replace("\\", "/") # Windows separator fix
inner_path = re.sub("^%s/" % re.escape(self.directory), "", inner_path) # Remove site directory if begins with it
file_path = u"%s/%s" % (self.directory, inner_path)
if not inner_path:
return self.directory
file_abspath = os.path.dirname(os.path.abspath(file_path))
if ".." in file_path or not file_abspath.startswith(self.allowed_dir):
raise Exception(u"File not allowed: %s" % file_path)
@ -222,7 +238,10 @@ class SiteStorage:
# Get site dir relative path
def getInnerPath(self, path):
inner_path = re.sub("^%s/" % re.escape(self.directory), "", path)
if path == self.directory:
inner_path = ""
else:
inner_path = re.sub("^%s/" % re.escape(self.directory), "", path)
return inner_path
# Verify all files sha512sum using content.json

View file

@ -22,7 +22,7 @@ class TestFileRequest:
# Add file_server as peer to client
peer_file_server = site_temp.addPeer("127.0.0.1", 1544)
assert peer_file_server.ping()
assert peer_file_server.ping() is not None
assert peer_file_server in site_temp.peers.values()
peer_file_server.remove()

View file

@ -62,3 +62,36 @@ class TestSite:
assert new_site.address in SiteManager.site_manager.sites
SiteManager.site_manager.delete(new_site.address)
assert new_site.address not in SiteManager.site_manager.sites
@pytest.mark.parametrize("inner_path", ["content.json", "data/test_include/content.json", "data/users/content.json"])
def testSign(self, site, inner_path):
# Bad privatekey
assert not site.content_manager.sign(inner_path, privatekey="5aaa3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMnaa", filewrite=False)
# Good privatekey
content = site.content_manager.sign(inner_path, privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
content_old = site.content_manager.contents[inner_path] # Content before the sign
assert not content_old == content # Timestamp changed
assert site.address in content["signs"] # Used the site's private key to sign
if inner_path == "content.json":
assert len(content["files"]) == 24
elif inner_path == "data/test-include/content.json":
assert len(content["files"]) == 1
elif inner_path == "data/users/content.json":
assert len(content["files"]) == 0
# Everything should be same as before except the modified timestamp and the signs
assert (
{key: val for key, val in content_old.items() if key not in ["modified", "signs", "sign"]}
==
{key: val for key, val in content.items() if key not in ["modified", "signs", "sign"]}
)
def testSignOptionalFiles(self, site):
site.content_manager.contents["content.json"]["optional"] = "((data/img/zero.*))"
content_optional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
del site.content_manager.contents["content.json"]["optional"]
content_nooptional = site.content_manager.sign(privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", filewrite=False)
assert len(content_nooptional["files"]) > len(content_optional["files"])

View file

@ -0,0 +1,15 @@
import shutil
import os
import pytest
from Site import SiteManager
@pytest.mark.usefixtures("resetSettings")
class TestSiteStorage:
def testList(self, site):
list_root = list(site.storage.list(""))
assert "content.json" in list_root
assert "css/all.css" in list_root
assert list(site.storage.list("data-default")) == ["data.json", "users/content-default.json"]

View file

@ -1,133 +1,133 @@
{
"address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT",
"background-color": "white",
"description": "Blogging platform Demo",
"domain": "Blog.ZeroNetwork.bit",
"address": "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT",
"background-color": "white",
"description": "Blogging platform Demo",
"domain": "Blog.ZeroNetwork.bit",
"files": {
"css/all.css": {
"sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc",
"sha512": "65ddd3a2071a0f48c34783aa3b1bde4424bdea344630af05a237557a62bd55dc",
"size": 112710
},
},
"data-default/data.json": {
"sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2",
"sha512": "3f5c5a220bde41b464ab116cce0bd670dd0b4ff5fe4a73d1dffc4719140038f2",
"size": 196
},
},
"data-default/users/content-default.json": {
"sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa",
"sha512": "0603ce08f7abb92b3840ad0cf40e95ea0b3ed3511b31524d4d70e88adba83daa",
"size": 679
},
},
"data/data.json": {
"sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52",
"sha512": "0f2321c905b761a05c360a389e1de149d952b16097c4ccf8310158356e85fb52",
"size": 31126
},
},
"data/img/autoupdate.png": {
"sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71",
"sha512": "d2b4dc8e0da2861ea051c0c13490a4eccf8933d77383a5b43de447c49d816e71",
"size": 24460
},
},
"data/img/direct_domains.png": {
"sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719",
"sha512": "5f14b30c1852735ab329b22496b1e2ea751cb04704789443ad73a70587c59719",
"size": 16185
},
},
"data/img/domain.png": {
"sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a",
"sha512": "ce87e0831f4d1e95a95d7120ca4d33f8273c6fce9f5bbedf7209396ea0b57b6a",
"size": 11881
},
},
"data/img/memory.png": {
"sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e",
"sha512": "dd56515085b4a79b5809716f76f267ec3a204be3ee0d215591a77bf0f390fa4e",
"size": 12775
},
},
"data/img/multiuser.png": {
"sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6",
"sha512": "88e3f795f9b86583640867897de6efc14e1aa42f93e848ed1645213e6cc210c6",
"size": 29480
},
},
"data/img/progressbar.png": {
"sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf",
"sha512": "23d592ae386ce14158cec34d32a3556771725e331c14d5a4905c59e0fe980ebf",
"size": 13294
},
},
"data/img/slides.png": {
"sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e",
"sha512": "1933db3b90ab93465befa1bd0843babe38173975e306286e08151be9992f767e",
"size": 14439
},
},
"data/img/slots_memory.png": {
"sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29",
"sha512": "82a250e6da909d7f66341e5b5c443353958f86728cd3f06e988b6441e6847c29",
"size": 9488
},
},
"data/img/trayicon.png": {
"sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f",
"sha512": "e7ae65bf280f13fb7175c1293dad7d18f1fcb186ebc9e1e33850cdaccb897b8f",
"size": 19040
},
},
"data/img/zeroblog-comments.png": {
"sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4",
"sha512": "efe4e815a260e555303e5c49e550a689d27a8361f64667bd4a91dbcccb83d2b4",
"size": 24001
},
},
"data/img/zeroid.png": {
"sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b",
"sha512": "b46d541a9e51ba2ddc8a49955b7debbc3b45fd13467d3c20ef104e9d938d052b",
"size": 18875
},
},
"data/img/zeroname.png": {
"sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d",
"sha512": "bab45a1bb2087b64e4f69f756b2ffa5ad39b7fdc48c83609cdde44028a7a155d",
"size": 36031
},
},
"data/img/zerotalk-mark.png": {
"sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af",
"sha512": "a335b2fedeb8d291ca68d3091f567c180628e80f41de4331a5feb19601d078af",
"size": 44862
},
},
"data/img/zerotalk-upvote.png": {
"sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732",
"sha512": "b1ffd7f948b4f99248dde7efe256c2efdfd997f7e876fb9734f986ef2b561732",
"size": 41092
},
},
"data/img/zerotalk.png": {
"sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8",
"sha512": "54d10497a1ffca9a4780092fd1bd158c15f639856d654d2eb33a42f9d8e33cd8",
"size": 26606
},
},
"data/test_include/data.json": {
"sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906",
"sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906",
"size": 505
},
},
"dbschema.json": {
"sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8",
"sha512": "7b756e8e475d4d6b345a24e2ae14254f5c6f4aa67391a94491a026550fe00df8",
"size": 1529
},
},
"img/loading.gif": {
"sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00",
"sha512": "8a42b98962faea74618113166886be488c09dad10ca47fe97005edc5fb40cc00",
"size": 723
},
},
"index.html": {
"sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3",
"sha512": "c4039ebfc4cb6f116cac05e803a18644ed70404474a572f0d8473f4572f05df3",
"size": 4667
},
},
"js/all.js": {
"sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c",
"sha512": "034c97535f3c9b3fbebf2dcf61a38711dae762acf1a99168ae7ddc7e265f582c",
"size": 201178
}
},
"ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)",
},
"ignore": "((js|css)/(?!all.(js|css))|data/.*db|data/users/.*/.*)",
"includes": {
"data/test_include/content.json": {
"added": 1424976057,
"files_allowed": "data.json",
"includes_allowed": false,
"max_size": 20000,
"signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ],
"signers_required": 1,
"user_id": 47,
"added": 1424976057,
"files_allowed": "data.json",
"includes_allowed": false,
"max_size": 20000,
"signers": [ "15ik6LeBWnACWfaika1xqGapRZ1zh3JpCo" ],
"signers_required": 1,
"user_id": 47,
"user_name": "test"
},
},
"data/users/content.json": {
"signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ],
"signers": [ "1LSxsKfC9S9TVXGGNSM3vPHjyW82jgCX5f" ],
"signers_required": 1
}
},
"modified": 1443088239.123,
},
"modified": 1443393859.801,
"sign": [
37796247323133993908968541760020085519225012317332056166386012116450888757672,
8182016604193300184892407269063757269964429504791487428802219119125679030316
],
"signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=",
30041653970398729892154852118727733790145614202537425646336077462070348808967,
96823925597554846684463773054016176426938620086211253074026312396122955360853
],
"signers_sign": "HDNmWJHM2diYln4pkdL+qYOvgE7MdwayzeG+xEUZBgp1HtOjBJS+knDEVQsBkjcOPicDG2it1r6R1eQrmogqSP0=",
"signs": {
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HJ+SuvmYh1DIyvqypUobaspZ3heUfYWoN34S4c2la5NgcBmpZ/YN4Xzi6wtP20W8DePXdsYMC0Azr+L8ZF7FAk4="
},
"signs_required": 1,
"title": "ZeroBlog",
"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": "HKBBvaOi1v20ZuSVORtD4bBRRgf/85QDVy4HaaPX3fFDAKYmvUWK+Jbp3yIGElMmPoO2+YljFLyromAoEwWd6Eg="
},
"signs_required": 1,
"title": "ZeroBlog",
"zeronet_version": "0.3.2"
}