oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rev470, Keep track downloaded optional files in hashfield, Peer getHashfield command, Check optional files on verifyFiles, Test hashfield, Test hashfield exchange

Browse source
This commit is contained in:
HelloZeroNet 2015-10-11 02:22:53 +02:00
parent f8fd58866b
commit 9400e9f58f
10 changed files with 178 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Config.py
View file

@ -8,7 +8,7 @@ class Config(object):
def __init__(self, argv):
self.version = "0.3.2"
self.rev = 467
self.rev = 470
self.argv = argv
self.action = None
self.createParser()

18
src/Content/ContentManager.py
View file

@ -10,6 +10,8 @@ from Debug import Debug
from Crypt import CryptHash
from Config import config
from util import helper
from Peer import PeerHashfield
class ContentManager(object):
@ -17,6 +19,8 @@ class ContentManager(object):
self.site = site
self.log = self.site.log
self.contents = {} # Known content.json (without files and includes)
self.hashfield = PeerHashfield()
self.site.onFileDone.append(lambda inner_path: self.addOptionalFile(inner_path))
self.loadContent(add_bad_files=False, delete_removed_files=False)
self.site.settings["size"] = self.getTotalSize()
@ -302,7 +306,9 @@ class ContentManager(object):
self.log.info("Opening site data directory: %s..." % directory)
changed_files = [inner_path]
files_node, files_optional_node = self.hashFiles(helper.getDirname(inner_path), content.get("ignore"), content.get("optional"))
files_node, files_optional_node = self.hashFiles(
helper.getDirname(inner_path), content.get("ignore"), content.get("optional")
)
# Find changed files
files_merged = files_node.copy()
@ -459,7 +465,9 @@ class ContentManager(object):
if rules.get("max_size_optional") is not None: # Include optional files limit
if content_size_optional > rules["max_size_optional"]:
self.log.error("%s: Include optional files too large %s > %s" % (inner_path, content_size_optional, rules["max_size_optional"]))
self.log.error("%s: Include optional files too large %s > %s" % (
inner_path, content_size_optional, rules["max_size_optional"])
)
return False
# Filename limit
@ -567,6 +575,12 @@ class ContentManager(object):
self.log.error("File not in content.json: %s" % inner_path)
return False
def addOptionalFile(self, inner_path):
info = self.getFileInfo(inner_path)
if info and info["optional"]:
self.log.debug("Downloaded optional file, adding to hashfield: %s" % inner_path)
self.hashfield.appendHash(info["sha512"])
if __name__ == "__main__":
def testSign():

16
src/File/FileRequest.py
View file

@ -65,6 +65,8 @@ class FileRequest(object):
self.actionPex(params)
elif cmd == "listModified":
self.actionListModified(params)
elif cmd == "getHashfield":
self.actionGetHashfield(params)
elif cmd == "ping":
self.actionPing()
else:
@ -257,6 +259,20 @@ class FileRequest(object):
self.response({"modified_files": modified_files})
def actionGetHashfield(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
if connected_peer: # Just added
connected_peer.connect(self.connection) # Assign current connection to peer
self.response({"hashfield_raw": site.content_manager.hashfield.tostring()})
# Send a simple Pong! answer
def actionPing(self):
self.response("Pong!")

15
src/Peer/Peer.py
View file

@ -8,6 +8,7 @@ from cStringIO import StringIO
from Debug import Debug
from Config import config
from util import helper
from PeerHashfield import PeerHashfield
if config.use_tempfiles:
import tempfile
@ -27,8 +28,8 @@ class Peer(object):
self.key = "%s:%s" % (ip, port)
self.connection = None
self.hashfield = array.array("H") # Got optional files hash_id
self.last_hashfield = None # Last time hashfiled downloaded
self.hashfield = PeerHashfield() # Got optional files hash_id
self.last_hashfield = 0 # Last time hashfiled downloaded
self.last_found = time.time() # Time of last found in the torrent tracker
self.last_response = None # Time of last successful response from peer
self.last_ping = None # Last response time for ping
@ -221,6 +222,16 @@ class Peer(object):
self.log("Added peers using pex: %s" % added)
return added
# Request optional files hashfield from peer
def getHashfield(self):
self.last_hashfield = time.time()
res = self.request("getHashfield", {"site": self.site.address})
if res and "error" not in res:
self.hashfield.replaceFromString(res["hashfield_raw"])
return self.hashfield
else:
return False
# List modified files since the date
# Return: {inner_path: modification date,...}
def listModified(self, since):

42
src/Peer/PeerHashfield.py Normal file
View file

@ -0,0 +1,42 @@
import array
class PeerHashfield():
def __init__(self):
self.storage = self.createStoreage()
def createStoreage(self):
storage = array.array("H")
self.append = storage.append
self.remove = storage.remove
self.tostring = storage.tostring
self.fromstring = storage.fromstring
self.__len__ = storage.__len__
self.__iter__ = storage.__iter__
return storage
def appendHash(self, hash):
hash_id = int(hash[0:4], 16)
if hash_id not in self:
self.append(hash_id)
return True
else:
return False
def removeHash(self, hash):
hash_id = int(hash[0:4], 16)
if hash_id in self:
self.remove(hash_id)
return True
else:
return False
def getHashId(self, hash):
return int(hash[0:4], 16)
def hasHash(self, hash):
return int(hash[0:4], 16) in self
def replaceFromString(self, hashfield_raw):
self.storage = self.createStoreage()
self.fromstring(hashfield_raw)

1
src/Peer/__init__.py
View file

@ -1 +1,2 @@
from Peer import Peer
from PeerHashfield import PeerHashfield

4
src/Site/Site.py
View file

@ -34,6 +34,7 @@ class Site:
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.addEventListeners()
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
@ -66,9 +67,6 @@ class Site:
self.websockets = [] # Active site websocket connections
# Add event listeners
self.addEventListeners()
def __str__(self):
return "Site %s" % self.address_short

37
src/Site/SiteStorage.py
View file

@ -218,6 +218,10 @@ class SiteStorage:
def isFile(self, inner_path):
return os.path.isfile(self.getPath(inner_path))
# File or directory exist
def isExists(self, inner_path):
return os.path.exists(self.getPath(inner_path))
# Dir exist
def isDir(self, inner_path):
return os.path.isdir(self.getPath(inner_path))
@ -246,6 +250,7 @@ class SiteStorage:
# Verify all files sha512sum using content.json
def verifyFiles(self, quick_check=False): # Fast = using file size
bad_files = []
if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first
self.site.needFile("content.json", update=True) # Force update to fix corrupt file
self.site.content_manager.loadContent() # Reload content.json
@ -253,7 +258,8 @@ class SiteStorage:
if not os.path.isfile(self.getPath(content_inner_path)): # Missing content.json file
self.log.debug("[MISSING] %s" % content_inner_path)
bad_files.append(content_inner_path)
for file_relative_path in content["files"].keys():
for file_relative_path in content.get("files", {}).keys():
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
file_inner_path = file_inner_path.strip("/") # Strip leading /
file_path = self.getPath(file_inner_path)
@ -270,9 +276,34 @@ class SiteStorage:
if not ok:
self.log.debug("[CHANGED] %s" % file_inner_path)
bad_files.append(file_inner_path)
# Optional files
optional_added = 0
optional_removed = 0
for file_relative_path in content.get("files_optional", {}).keys():
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
file_inner_path = file_inner_path.strip("/") # Strip leading /
file_path = self.getPath(file_inner_path)
if not os.path.isfile(file_path):
self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"])
continue
if quick_check:
ok = os.path.getsize(file_path) == content["files_optional"][file_relative_path]["size"]
else:
ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb"))
if ok:
self.site.content_manager.hashfield.appendHash(content["files_optional"][file_relative_path]["sha512"])
optional_added += 1
else:
self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"])
optional_removed += 1
self.log.debug("[OPTIONAL CHANGED] %s" % file_inner_path)
self.log.debug(
"%s verified: %s files, quick_check: %s, bad files: %s" %
(content_inner_path, len(content["files"]), quick_check, bad_files)
"%s verified: %s, quick: %s, bad: %s, optionals: +%s -%s" %
(content_inner_path, len(content["files"]), quick_check, bad_files, optional_added, optional_removed)
)
return bad_files

50
src/Test/TestPeer.py
View file

@ -1,11 +1,9 @@
import cStringIO as StringIO
import pytest
import time
from Connection import ConnectionServer
from Connection import Connection
from File import FileServer
from Crypt import CryptHash
from cStringIO import StringIO
@pytest.mark.usefixtures("resetSettings")
@ -52,3 +50,47 @@ class TestFileRequest:
connection.close()
client.stop()
def testHashfield(self, site):
sample_hash = site.content_manager.contents["content.json"]["files_optional"].values()[0]["sha512"]
site.storage.verifyFiles(quick_check=True) # Find what optional files we have
# Check if hashfield has any files
assert len(site.content_manager.hashfield) > 0
# Check exsist hash
assert site.content_manager.hashfield.getHashId(sample_hash) in site.content_manager.hashfield
# Add new hash
new_hash = CryptHash.sha512sum(StringIO("hello"))
assert site.content_manager.hashfield.getHashId(new_hash) not in site.content_manager.hashfield
assert site.content_manager.hashfield.appendHash(new_hash)
assert not site.content_manager.hashfield.appendHash(new_hash) # Don't add second time
assert site.content_manager.hashfield.getHashId(new_hash) in site.content_manager.hashfield
# Remove new hash
assert site.content_manager.hashfield.removeHash(new_hash)
assert site.content_manager.hashfield.getHashId(new_hash) not in site.content_manager.hashfield
def testHashfieldExchange(self, file_server, site, site_temp):
file_server.ip_incoming = {} # Reset flood protection
file_server.sites[site.address] = site
client = FileServer("127.0.0.1", 1545)
client.sites[site_temp.address] = site_temp
site_temp.connection_server = client
connection = client.getConnection("127.0.0.1", 1544)
site.storage.verifyFiles(quick_check=True) # Find what optional files we have
# Add file_server as peer to client
peer_file_server = site_temp.addPeer("127.0.0.1", 1544)
# Check if hashfield has any files
assert len(site.content_manager.hashfield) > 0
# Testing hashfield sync
assert len(peer_file_server.hashfield) == 0
assert peer_file_server.getHashfield()
assert len(peer_file_server.hashfield) > 0
connection.close()
client.stop()

11
src/Test/TestSiteDownload.py
View file

@ -1,16 +1,15 @@
import pytest
import mock
import time
from Connection import ConnectionServer
from Config import config
from Site import Site
from File import FileRequest
import Spy
@pytest.mark.usefixtures("resetTempSettings")
@pytest.mark.usefixtures("resetSettings")
class TestWorker:
class TestSiteDownload:
def testDownload(self, file_server, site, site_temp):
client = ConnectionServer("127.0.0.1", 1545)
assert site.storage.directory == config.data_dir + "/" + site.address
@ -57,7 +56,13 @@ class TestWorker:
# Optional user file
assert not site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif")
optional_file_info = site_temp.content_manager.getFileInfo(
"data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif"
)
assert not site_temp.content_manager.hashfield.hasHash(optional_file_info["sha512"])
site_temp.needFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif")
assert site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif")
assert site_temp.content_manager.hashfield.hasHash(optional_file_info["sha512"])
assert site_temp.storage.deleteFiles()