oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #119 from mattseh/cleanup

Browse source 
Changing to PEP8 coding style
This commit is contained in:
ZeroNet 2015-06-18 01:42:49 +02:00
commit c565a12fc3
9 changed files with 1278 additions and 1282 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
src/Content/ContentManager.py
View file

@ -3,7 +3,7 @@ from Debug import Debug
from Crypt import CryptHash from Crypt import CryptHash
from Config import config from Config import config
class ContentManager: class ContentManager(object):
def __init__(self, site): def __init__(self, site):
self.site = site self.site = site
self.log = self.site.log self.log = self.site.log
@ -11,7 +11,6 @@ class ContentManager:
self.loadContent(add_bad_files=False) self.loadContent(add_bad_files=False)
self.site.settings["size"] = self.getTotalSize() self.site.settings["size"] = self.getTotalSize()
# Load content.json to self.content # Load content.json to self.content
# Return: Changed files ["index.html", "data/messages.json"] # Return: Changed files ["index.html", "data/messages.json"]
def loadContent(self, content_inner_path="content.json", add_bad_files=True, load_includes=True): def loadContent(self, content_inner_path="content.json", add_bad_files=True, load_includes=True):
@ -31,7 +30,6 @@ class ContentManager:
self.log.error("Content.json not exist: %s" % content_path) self.log.error("Content.json not exist: %s" % content_path)
return False # Content.json not exist return False # Content.json not exist
try: try:
# Get the files where the sha512 changed # Get the files where the sha512 changed
changed = [] changed = []
@ -83,7 +81,6 @@ class ContentManager:
return changed return changed
# Get total size of site # Get total size of site
# Return: 32819 (size of files in kb) # Return: 32819 (size of files in kb)
def getTotalSize(self, ignore=None): def getTotalSize(self, ignore=None):
@ -95,7 +92,6 @@ class ContentManager:
total_size += info["size"] total_size += info["size"]
return total_size return total_size
# Find the file info line from self.contents # Find the file info line from self.contents
# Return: { "sha512": "c29d73d30ee8c9c1b5600e8a84447a6de15a3c3db6869aca4a2a578c1721f518", "size": 41 , "content_inner_path": "content.json"} # Return: { "sha512": "c29d73d30ee8c9c1b5600e8a84447a6de15a3c3db6869aca4a2a578c1721f518", "size": 41 , "content_inner_path": "content.json"}
def getFileInfo(self, inner_path): def getFileInfo(self, inner_path):
@ -112,7 +108,8 @@ class ContentManager:
if content and "user_contents" in content: # User dir if content and "user_contents" in content: # User dir
back = content["user_contents"] back = content["user_contents"]
back["content_inner_path"] = re.sub("(.*)/.*?$", "\\1/content.json", inner_path) # Content.json is in the users dir # Content.json is in the users dir
back["content_inner_path"] = re.sub("(.*)/.*?$", "\\1/content.json", inner_path)
return back return back
# No inner path in this dir, lets try the parent dir # No inner path in this dir, lets try the parent dir
@ -121,8 +118,8 @@ class ContentManager:
else: # No more parent dirs else: # No more parent dirs
break break
return False # Not found # Not found
return False
# Get rules for the file # Get rules for the file
# Return: The rules for the file or False if not allowed # Return: The rules for the file or False if not allowed
@ -157,8 +154,8 @@ class ContentManager:
user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1) # Delivered for directory user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1) # Delivered for directory
try: try:
if not content: content = self.site.storage.loadJson(inner_path) # Read the file if no content specificed if not content: content = self.site.storage.loadJson(inner_path) # Read the file if no content specified
except: # Content.json not exist except (Exception, ): # Content.json not exist
return {"signers": [user_address], "user_address": user_address} # Return information that we know for sure return {"signers": [user_address], "user_address": user_address} # Return information that we know for sure
"""if not "cert_user_name" in content: # New file, unknown user """if not "cert_user_name" in content: # New file, unknown user
@ -168,8 +165,10 @@ class ContentManager:
user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/nofish@zeroid.bit user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/nofish@zeroid.bit
rules = copy.copy(user_contents["permissions"].get(content["cert_user_id"], {})) # Default rules by username rules = copy.copy(user_contents["permissions"].get(content["cert_user_id"], {})) # Default rules by username
if rules == False: return False # User banned if not rules:
if "signers" in rules: rules["signers"] = rules["signers"][:] # Make copy of the signers return False # User banned
if "signers" in rules:
rules["signers"] = rules["signers"][:] # Make copy of the signers
for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules
if not re.match(permission_pattern, user_urn): continue # Rule is not valid for user if not re.match(permission_pattern, user_urn): continue # Rule is not valid for user
# Update rules if its better than current recorded ones # Update rules if its better than current recorded ones
@ -180,7 +179,8 @@ class ContentManager:
else: else:
rules[key] = val rules[key] = val
elif type(val) is int: # Int, update if larger elif type(val) is int: # Int, update if larger
if val > rules[key]: rules[key] = val if val > rules[key]:
rules[key] = val
elif hasattr(val, "startswith"): # String, update if longer elif hasattr(val, "startswith"): # String, update if longer
if len(val) > len(rules[key]): rules[key] = val if len(val) > len(rules[key]): rules[key] = val
elif type(val) is list: # List, append elif type(val) is list: # List, append
@ -191,11 +191,8 @@ class ContentManager:
rules["signers"].append(user_address) # Add user as valid signer rules["signers"].append(user_address) # Add user as valid signer
rules["user_address"] = user_address rules["user_address"] = user_address
return rules return rules
# Create and sign a content.json # Create and sign a content.json
# Return: The new content if filewrite = False # Return: The new content if filewrite = False
def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None): def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None):
@ -203,7 +200,7 @@ class ContentManager:
if not content: # Content not exist yet, load default one if not content: # Content not exist yet, load default one
self.log.info("File %s not exist yet, loading default values..." % inner_path) self.log.info("File %s not exist yet, loading default values..." % inner_path)
content = {"files": {}, "signs": {}} # Default content.json content = {"files": {}, "signs": {}} # Default content.json
if inner_path == "content.json": # Its the root content.json, add some more fields if inner_path == "content.json": # It's the root content.json, add some more fields
content["title"] = "%s - ZeroNet_" % self.site.address content["title"] = "%s - ZeroNet_" % self.site.address
content["description"] = "" content["description"] = ""
content["signs_required"] = 1 content["signs_required"] = 1
@ -234,7 +231,6 @@ class ContentManager:
if file_inner_path in content["files"].keys() and hashed_files[file_inner_path]["sha512"] != content["files"][file_inner_path].get("sha512"): if file_inner_path in content["files"].keys() and hashed_files[file_inner_path]["sha512"] != content["files"][file_inner_path].get("sha512"):
changed_files.append(file_path) changed_files.append(file_path)
self.log.debug("Changed files: %s" % changed_files) self.log.debug("Changed files: %s" % changed_files)
if update_changed_files: if update_changed_files:
for file_path in changed_files: for file_path in changed_files:
@ -294,7 +290,6 @@ class ContentManager:
else: # Return the new content else: # Return the new content
return new_content return new_content
# The valid signers of content.json file # The valid signers of content.json file
# Return: ["1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6", "13ReyhCsjhpuCVahn1DHdf6eMqqEVev162"] # Return: ["1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6", "13ReyhCsjhpuCVahn1DHdf6eMqqEVev162"]
def getValidSigners(self, inner_path, content=None): def getValidSigners(self, inner_path, content=None):
@ -307,15 +302,14 @@ class ContentManager:
if rules and "signers" in rules: if rules and "signers" in rules:
valid_signers += rules["signers"] valid_signers += rules["signers"]
if self.site.address not in valid_signers: valid_signers.append(self.site.address) # Site address always valid if self.site.address not in valid_signers:
valid_signers.append(self.site.address) # Site address always valid
return valid_signers return valid_signers
# Return: The required number of valid signs for the content.json # Return: The required number of valid signs for the content.json
def getSignsRequired(self, inner_path, content=None): def getSignsRequired(self, inner_path, content=None):
return 1 # Todo: Multisig return 1 # Todo: Multisig
def verifyCert(self, inner_path, content): def verifyCert(self, inner_path, content):
from Crypt import CryptBitcoin from Crypt import CryptBitcoin
@ -329,7 +323,6 @@ class ContentManager:
return False return False
return CryptBitcoin.verify("%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name), cert_address, content["cert_sign"]) return CryptBitcoin.verify("%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name), cert_address, content["cert_sign"])
# Checks if the content.json content is valid # Checks if the content.json content is valid
# Return: True or False # Return: True or False
def validContent(self, inner_path, content): def validContent(self, inner_path, content):
@ -375,8 +368,6 @@ class ContentManager:
return True # All good return True # All good
# Verify file validity # Verify file validity
# Return: None = Same as before, False = Invalid, True = Valid # Return: None = Same as before, False = Invalid, True = Valid
def verifyFile(self, inner_path, file, ignore_same = True): def verifyFile(self, inner_path, file, ignore_same = True):
@ -443,7 +434,8 @@ class ContentManager:
else: else:
hash_valid = False hash_valid = False
if file_info["size"] != file.tell(): if file_info["size"] != file.tell():
self.log.error("%s file size does not match %s <> %s, Hash: %s" % (inner_path, file.tell(), file_info["size"], hash_valid)) self.log.error("%s file size does not match %s <> %s, Hash: %s" % (inner_path, file.tell(),
file_info["size"], hash_valid))
return False return False
return hash_valid return hash_valid
@ -460,8 +452,6 @@ class ContentManager:
return file_dir return file_dir
def testSign(): def testSign():
global config global config
from Config import config from Config import config

47
src/File/FileRequest.py
View file

@ -1,5 +1,12 @@
import os, msgpack, shutil, gevent, socket, struct, random # Included modules
import socket
import struct
import os
from cStringIO import StringIO from cStringIO import StringIO
# Third party modules
import gevent
from Debug import Debug from Debug import Debug
from Config import config from Config import config
from util import RateLimit, StreamingMsgpack from util import RateLimit, StreamingMsgpack
@ -19,16 +26,13 @@ class FileRequest(object):
self.log = server.log self.log = server.log
self.responded = False # Responded to the request self.responded = False # Responded to the request
def unpackAddress(self, packed): def unpackAddress(self, packed):
return (socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0]) return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0]
def send(self, msg, streaming=False): def send(self, msg, streaming=False):
if not self.connection.closed: if not self.connection.closed:
self.connection.send(msg, streaming) self.connection.send(msg, streaming)
def response(self, msg, streaming=False): def response(self, msg, streaming=False):
if self.responded: if self.responded:
self.log.debug("Req id %s already responded" % self.req_id) self.log.debug("Req id %s already responded" % self.req_id)
@ -40,7 +44,6 @@ class FileRequest(object):
self.responded = True self.responded = True
self.send(msg, streaming=streaming) self.send(msg, streaming=streaming)
# Route file requests # Route file requests
def route(self, cmd, req_id, params): def route(self, cmd, req_id, params):
self.req_id = req_id self.req_id = req_id
@ -49,9 +52,10 @@ class FileRequest(object):
self.actionGetFile(params) self.actionGetFile(params)
elif cmd == "update": elif cmd == "update":
event = "%s update %s %s" % (self.connection.id, params["site"], params["inner_path"]) event = "%s update %s %s" % (self.connection.id, params["site"], params["inner_path"])
if not RateLimit.isAllowed(event): # There was already an updat for this file in the last 10 second if not RateLimit.isAllowed(event): # There was already an update for this file in the last 10 second
self.response({"ok": "File update queued"}) self.response({"ok": "File update queued"})
RateLimit.callAsync(event, 10, self.actionUpdate, params) # If called more than once within 10 sec only keep the last update # If called more than once within 10 sec only keep the last update
RateLimit.callAsync(event, 10, self.actionUpdate, params)
elif cmd == "pex": elif cmd == "pex":
self.actionPex(params) self.actionPex(params)
@ -62,7 +66,6 @@ class FileRequest(object):
else: else:
self.actionUnknown(cmd, params) self.actionUnknown(cmd, params)
# Update a site file request # Update a site file request
def actionUpdate(self, params): def actionUpdate(self, params):
site = self.sites.get(params["site"]) site = self.sites.get(params["site"])
@ -105,7 +108,6 @@ class FileRequest(object):
self.log.debug("Update for %s is invalid" % params["inner_path"]) self.log.debug("Update for %s is invalid" % params["inner_path"])
self.response({"error": "File invalid"}) self.response({"error": "File invalid"})
# Send file content request # Send file content request
def actionGetFile(self, params): def actionGetFile(self, params):
site = self.sites.get(params["site"]) site = self.sites.get(params["site"])
@ -118,13 +120,17 @@ class FileRequest(object):
with StreamingMsgpack.FilePart(file_path, "rb") as file: with StreamingMsgpack.FilePart(file_path, "rb") as file:
file.seek(params["location"]) file.seek(params["location"])
file.read_bytes = FILE_BUFF file.read_bytes = FILE_BUFF
back = {} back = {"body": file,
back["body"] = file "size": os.fstat(file.fileno()).st_size,
back["size"] = os.fstat(file.fileno()).st_size "location": min(file.tell()+FILE_BUFF, os.fstat(file.fileno()).st_size)
back["location"] = min(file.tell()+FILE_BUFF, back["size"]) }
if config.debug_socket: self.log.debug("Sending file %s from position %s to %s" % (file_path, params["location"], back["location"])) if config.debug_socket:
self.log.debug("Sending file %s from position %s to %s" % (file_path,
params["location"],
back["location"]))
self.response(back, streaming=True) self.response(back, streaming=True)
if config.debug_socket: self.log.debug("File %s sent" % file_path) if config.debug_socket:
self.log.debug("File %s sent" % file_path)
# Add peer to site if not added before # Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port) connected_peer = site.addPeer(self.connection.ip, self.connection.port)
@ -136,7 +142,6 @@ class FileRequest(object):
self.response({"error": "File read error: %s" % Debug.formatException(err)}) self.response({"error": "File read error: %s" % Debug.formatException(err)})
return False return False
# Peer exchange request # Peer exchange request
def actionPex(self, params): def actionPex(self, params):
site = self.sites.get(params["site"]) site = self.sites.get(params["site"])
@ -162,14 +167,15 @@ class FileRequest(object):
self.log.debug("Added %s peers to %s using pex, sending back %s" % (added, site, len(packed_peers))) self.log.debug("Added %s peers to %s using pex, sending back %s" % (added, site, len(packed_peers)))
self.response({"peers": packed_peers}) self.response({"peers": packed_peers})
# Get modified content.json files since # Get modified content.json files since
def actionListModified(self, params): def actionListModified(self, params):
site = self.sites.get(params["site"]) site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"}) self.response({"error": "Unknown site"})
return False return False
modified_files = {inner_path: content["modified"] for inner_path, content in site.content_manager.contents.iteritems() if content["modified"] > params["since"]} modified_files = {inner_path: content["modified"]
for inner_path, content in site.content_manager.contents.iteritems()
if content["modified"] > params["since"]}
# Add peer to site if not added before # Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port) connected_peer = site.addPeer(self.connection.ip, self.connection.port)
@ -178,13 +184,10 @@ class FileRequest(object):
self.response({"modified_files": modified_files}) self.response({"modified_files": modified_files})
# Send a simple Pong! answer # Send a simple Pong! answer
def actionPing(self): def actionPing(self):
self.response("Pong!") self.response("Pong!")
# Unknown command # Unknown command
def actionUnknown(self, cmd, params): def actionUnknown(self, cmd, params):
self.response({"error": "Unknown command: %s" % cmd}) self.response({"error": "Unknown command: %s" % cmd})

49
src/Peer/Peer.py
View file

@ -5,7 +5,8 @@ from Debug import Debug
# Communicate remote peers # Communicate remote peers
class Peer(object): class Peer(object):
__slots__ = ("ip", "port", "site", "key", "connection_server", "connection", "last_found", "last_response", "last_ping", "added", "connection_error", "hash_failed", "download_bytes", "download_time") __slots__ = ("ip", "port", "site", "key", "connection_server", "connection", "last_found", "last_response",
"last_ping", "added", "connection_error", "hash_failed", "download_bytes", "download_time")
def __init__(self, ip, port, site=None): def __init__(self, ip, port, site=None):
self.ip = ip self.ip = ip
@ -16,7 +17,7 @@ class Peer(object):
self.connection = None self.connection = None
self.last_found = None # Time of last found in the torrent tracker self.last_found = None # Time of last found in the torrent tracker
self.last_response = None # Time of last successfull response from peer self.last_response = None # Time of last successful response from peer
self.last_ping = None # Last response time for ping self.last_ping = None # Last response time for ping
self.added = time.time() self.added = time.time()
@ -25,14 +26,12 @@ class Peer(object):
self.download_bytes = 0 # Bytes downloaded self.download_bytes = 0 # Bytes downloaded
self.download_time = 0 # Time spent to download self.download_time = 0 # Time spent to download
def log(self, text): def log(self, text):
if self.site: if self.site:
self.site.log.debug("%s:%s %s" % (self.ip, self.port, text)) self.site.log.debug("%s:%s %s" % (self.ip, self.port, text))
else: else:
logging.debug("%s:%s %s" % (self.ip, self.port, text)) logging.debug("%s:%s %s" % (self.ip, self.port, text))
# Connect to host # Connect to host
def connect(self, connection=None): def connect(self, connection=None):
if self.connection: if self.connection:
@ -41,7 +40,7 @@ class Peer(object):
else: else:
self.log("Getting connection...") self.log("Getting connection...")
if connection: # Connection specificed if connection: # Connection specified
self.connection = connection self.connection = connection
else: # Try to find from connection pool or create new connection else: # Try to find from connection pool or create new connection
self.connection = None self.connection = None
@ -53,7 +52,6 @@ class Peer(object):
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed)) self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed))
self.connection = None self.connection = None
# Check if we have connection to peer # Check if we have connection to peer
def findConnection(self): def findConnection(self):
if self.connection and self.connection.connected: # We have connection to peer if self.connection and self.connection.connected: # We have connection to peer
@ -62,28 +60,23 @@ class Peer(object):
self.connection = self.connection_server.getConnection(self.ip, self.port, create=False) # Do not create new connection if not found self.connection = self.connection_server.getConnection(self.ip, self.port, create=False) # Do not create new connection if not found
return self.connection return self.connection
def __str__(self): def __str__(self):
return "Peer %-12s" % self.ip return "Peer %-12s" % self.ip
def __repr__(self): def __repr__(self):
return "<%s>" % self.__str__() return "<%s>" % self.__str__()
# Peer ip:port to packed 6byte format # Peer ip:port to packed 6byte format
def packAddress(self): def packAddress(self):
return socket.inet_aton(self.ip)+struct.pack("H", self.port) return socket.inet_aton(self.ip)+struct.pack("H", self.port)
def unpackAddress(self, packed): def unpackAddress(self, packed):
return (socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0]) return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0]
# Found a peer on tracker # Found a peer on tracker
def found(self): def found(self):
self.last_found = time.time() self.last_found = time.time()
# Send a command to peer # Send a command to peer
def request(self, cmd, params={}): def request(self, cmd, params={}):
if not self.connection or self.connection.closed: if not self.connection or self.connection.closed:
@ -99,7 +92,8 @@ class Peer(object):
#if config.debug_socket: self.log.debug("sendCmd: %s %s" % (cmd, params.get("inner_path"))) #if config.debug_socket: self.log.debug("sendCmd: %s %s" % (cmd, params.get("inner_path")))
try: try:
response = self.connection.request(cmd, params) response = self.connection.request(cmd, params)
if not response: raise Exception("Send error") if not response:
raise Exception("Send error")
#if config.debug_socket: self.log.debug("Got response to: %s" % cmd) #if config.debug_socket: self.log.debug("Got response to: %s" % cmd)
if "error" in response: if "error" in response:
self.log("%s error: %s" % (cmd, response["error"])) self.log("%s error: %s" % (cmd, response["error"]))
@ -109,23 +103,24 @@ class Peer(object):
self.last_response = time.time() self.last_response = time.time()
return response return response
except Exception, err: except Exception, err:
if type(err).__name__ == "Notify": # Greenlet kill by worker if type(err).__name__ == "Notify": # Greenlet killed by worker
self.log("Peer worker got killed: %s, aborting cmd: %s" % (err.message, cmd)) self.log("Peer worker got killed: %s, aborting cmd: %s" % (err.message, cmd))
break break
else: else:
self.onConnectionError() self.onConnectionError()
self.log("%s (connection_error: %s, hash_failed: %s, retry: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed, retry)) self.log("%s (connection_error: %s, hash_failed: %s, retry: %s)" % (Debug.formatException(err),
self.connection_error,
self.hash_failed, retry))
time.sleep(1*retry) time.sleep(1*retry)
self.connect() self.connect()
return None # Failed after 4 retry return None # Failed after 4 retry
# Get a file content from peer # Get a file content from peer
def getFile(self, site, inner_path): def getFile(self, site, inner_path):
location = 0 location = 0
buff = StringIO() buff = StringIO()
s = time.time() s = time.time()
while 1: # Read in 512k parts while True: # Read in 512k parts
back = self.request("getFile", {"site": site, "inner_path": inner_path, "location": location}) # Get file content from last location back = self.request("getFile", {"site": site, "inner_path": inner_path, "location": location}) # Get file content from last location
if not back or "body" not in back: # Error if not back or "body" not in back: # Error
return False return False
@ -141,13 +136,12 @@ class Peer(object):
buff.seek(0) buff.seek(0)
return buff return buff
# Send a ping request # Send a ping request
def ping(self): def ping(self):
response_time = None response_time = None
for retry in range(1, 3): # Retry 3 times for retry in range(1, 3): # Retry 3 times
s = time.time() s = time.time()
with gevent.Timeout(10.0, False): # 10 sec timeout, dont raise exception with gevent.Timeout(10.0, False): # 10 sec timeout, don't raise exception
response = self.request("ping") response = self.request("ping")
if response and "body" in response and response["body"] == "Pong!": if response and "body" in response and response["body"] == "Pong!":
@ -165,29 +159,28 @@ class Peer(object):
self.last_ping = response_time self.last_ping = response_time
return response_time return response_time
# Request peer exchange from peer # Request peer exchange from peer
def pex(self, site=None, need_num=5): def pex(self, site=None, need_num=5):
if not site: site = self.site # If no site definied request peers for this site if not site:
packed_peers = [peer.packAddress() for peer in self.site.getConnectablePeers(5)] # give him/her 5 connectable peers site = self.site # If no site defined request peers for this site
# give him/her 5 connectible peers
packed_peers = [peer.packAddress() for peer in self.site.getConnectablePeers(5)]
response = self.request("pex", {"site": site.address, "peers": packed_peers, "need": need_num}) response = self.request("pex", {"site": site.address, "peers": packed_peers, "need": need_num})
if not response or "error" in response: if not response or "error" in response:
return False return False
added = 0 added = 0
for peer in response.get("peers", []): for peer in response.get("peers", []):
address = self.unpackAddress(peer) address = self.unpackAddress(peer)
if (site.addPeer(*address)): added += 1 if site.addPeer(*address):
added += 1
if added: if added:
self.log("Added peers using pex: %s" % added) self.log("Added peers using pex: %s" % added)
return added return added
# List modified files since the date # List modified files since the date
# Return: {inner_path: modification date,...} # Return: {inner_path: modification date,...}
def listModified(self, since): def listModified(self, since):
response = self.request("listModified", {"since": since, "site": self.site.address}) return self.request("listModified", {"since": since, "site": self.site.address})
return response
# Stop and remove from site # Stop and remove from site
def remove(self): def remove(self):
@ -196,7 +189,6 @@ class Peer(object):
if self.connection: if self.connection:
self.connection.close() self.connection.close()
# - EVENTS - # - EVENTS -
# On connection error # On connection error
@ -205,7 +197,6 @@ class Peer(object):
if self.connection_error >= 3: # Dead peer if self.connection_error >= 3: # Dead peer
self.remove() self.remove()
# Done working with peer # Done working with peer
def onWorkerDone(self): def onWorkerDone(self):
pass pass

15
src/User/User.py
View file

@ -21,11 +21,11 @@ class User(object):
self.log = logging.getLogger("User:%s" % self.master_address) self.log = logging.getLogger("User:%s" % self.master_address)
# Save to data/users.json # Save to data/users.json
def save(self): def save(self):
users = json.load(open("%s/users.json" % config.data_dir)) users = json.load(open("%s/users.json" % config.data_dir))
if not self.master_address in users: users[self.master_address] = {} # Create if not exist if self.master_address not in users:
users[self.master_address] = {} # Create if not exist
user_data = users[self.master_address] user_data = users[self.master_address]
if self.master_seed: user_data["master_seed"] = self.master_seed if self.master_seed: user_data["master_seed"] = self.master_seed
user_data["sites"] = self.sites user_data["sites"] = self.sites
@ -33,15 +33,13 @@ class User(object):
open("%s/users.json" % config.data_dir, "w").write(json.dumps(users, indent=2, sort_keys=True)) open("%s/users.json" % config.data_dir, "w").write(json.dumps(users, indent=2, sort_keys=True))
self.log.debug("Saved") self.log.debug("Saved")
def getAddressAuthIndex(self, address): def getAddressAuthIndex(self, address):
return int(address.encode("hex"), 16) return int(address.encode("hex"), 16)
# Get user site data # Get user site data
# Return: {"auth_address": "xxx", "auth_privatekey": "xxx"} # Return: {"auth_address": "xxx", "auth_privatekey": "xxx"}
def getSiteData(self, address, create=True): def getSiteData(self, address, create=True):
if not address in self.sites: # Genreate new BIP32 child key based on site address if address not in self.sites: # Generate new BIP32 child key based on site address
if not create: return {"auth_address": None, "auth_privatekey": None} # Dont create user yet if not create: return {"auth_address": None, "auth_privatekey": None} # Dont create user yet
s = time.time() s = time.time()
address_id = self.getAddressAuthIndex(address) # Convert site address to int address_id = self.getAddressAuthIndex(address) # Convert site address to int
@ -54,7 +52,6 @@ class User(object):
self.log.debug("Added new site: %s in %.3fs" % (address, time.time()-s)) self.log.debug("Added new site: %s in %.3fs" % (address, time.time()-s))
return self.sites[address] return self.sites[address]
# Get data for a new, unique site # Get data for a new, unique site
# Return: [site_address, bip32_index, {"auth_address": "xxx", "auth_privatekey": "xxx", "privatekey": "xxx"}] # Return: [site_address, bip32_index, {"auth_address": "xxx", "auth_privatekey": "xxx", "privatekey": "xxx"}]
def getNewSiteData(self): def getNewSiteData(self):
@ -69,7 +66,6 @@ class User(object):
self.save() self.save()
return site_address, bip32_index, self.sites[site_address] return site_address, bip32_index, self.sites[site_address]
# Get BIP32 address from site address # Get BIP32 address from site address
# Return: BIP32 auth address # Return: BIP32 auth address
def getAuthAddress(self, address, create=True): def getAuthAddress(self, address, create=True):
@ -79,7 +75,6 @@ class User(object):
else: else:
return self.getSiteData(address, create)["auth_address"] return self.getSiteData(address, create)["auth_address"]
def getAuthPrivatekey(self, address, create=True): def getAuthPrivatekey(self, address, create=True):
cert = self.getCert(address) cert = self.getCert(address)
if cert: if cert:
@ -87,7 +82,6 @@ class User(object):
else: else:
return self.getSiteData(address, create)["auth_privatekey"] return self.getSiteData(address, create)["auth_privatekey"]
# Add cert for the user # Add cert for the user
def addCert(self, auth_address, domain, auth_type, auth_user_name, cert_sign): def addCert(self, auth_address, domain, auth_type, auth_user_name, cert_sign):
domain = domain.lower() domain = domain.lower()
@ -109,7 +103,6 @@ class User(object):
self.save() self.save()
return True return True
def setCert(self, address, domain): def setCert(self, address, domain):
site_data = self.getSiteData(address) site_data = self.getSiteData(address)
if domain: if domain:
@ -119,7 +112,6 @@ class User(object):
self.save() self.save()
return site_data return site_data
# Get cert for the site address # Get cert for the site address
# Return: { "auth_address": ..., "auth_privatekey":..., "auth_type": "web", "auth_user_name": "nofish", "cert_sign": ... } or None # Return: { "auth_address": ..., "auth_privatekey":..., "auth_type": "web", "auth_user_name": "nofish", "cert_sign": ... } or None
def getCert(self, address): def getCert(self, address):
@ -127,7 +119,6 @@ class User(object):
if not site_data or not "cert" in site_data: return None # Site dont have cert if not site_data or not "cert" in site_data: return None # Site dont have cert
return self.certs.get(site_data["cert"]) return self.certs.get(site_data["cert"])
# Get cert user name for the site address # Get cert user name for the site address
# Return: user@certprovider.bit or None # Return: user@certprovider.bit or None
def getCertUserId(self, address): def getCertUserId(self, address):

20
src/User/UserManager.py
View file

@ -1,4 +1,9 @@
import json, logging, os # Included modules
import os
import json
import logging
# ZeroNet Modules
from User import User from User import User
from Plugin import PluginManager from Plugin import PluginManager
from Config import config from Config import config
@ -9,10 +14,10 @@ class UserManager(object):
def __init__(self): def __init__(self):
self.users = {} self.users = {}
# Load all user from data/users.json # Load all user from data/users.json
def load(self): def load(self):
if not self.users: self.users = {} if not self.users:
self.users = {}
user_found = [] user_found = []
added = 0 added = 0
@ -30,8 +35,8 @@ class UserManager(object):
del(self.users[master_address]) del(self.users[master_address])
logging.debug("Removed user: %s" % master_address) logging.debug("Removed user: %s" % master_address)
if added: logging.debug("UserManager added %s users" % added) if added:
logging.debug("UserManager added %s users" % added)
# Create new user # Create new user
# Return: User # Return: User
@ -43,7 +48,6 @@ class UserManager(object):
user.save() user.save()
return user return user
# List all users from data/users.json # List all users from data/users.json
# Return: {"usermasteraddr": User} # Return: {"usermasteraddr": User}
def list(self): def list(self):
@ -51,7 +55,6 @@ class UserManager(object):
self.load() self.load()
return self.users return self.users
# Get user based on master_address # Get user based on master_address
# Return: User or None # Return: User or None
def get(self, master_address=None): def get(self, master_address=None):
@ -62,7 +65,8 @@ class UserManager(object):
return None return None
user_manager = UserManager() # Singletone user_manager = UserManager() # Singleton
# Debug: Reload User.py # Debug: Reload User.py
def reloadModule(): def reloadModule():

2
src/Worker/Worker.py
View file

@ -2,7 +2,7 @@ import gevent, time, logging, shutil, os
from Peer import Peer from Peer import Peer
from Debug import Debug from Debug import Debug
class Worker: class Worker(object):
def __init__(self, manager, peer): def __init__(self, manager, peer):
self.manager = manager self.manager = manager
self.peer = peer self.peer = peer

54
src/main.py
View file

@ -1,4 +1,18 @@
import os, sys # Included modules
import os
import sys
import time
import urllib2
# Third party modules
import gevent
from gevent import monkey
# ZeroNet modules
import logging
update_after_shutdown = False # If set True then update and restart zeronet after main loop ended update_after_shutdown = False # If set True then update and restart zeronet after main loop ended
# Load config # Load config
@ -7,17 +21,21 @@ from Config import config
# Create necessary files and dirs # Create necessary files and dirs
if not os.path.isdir(config.log_dir): os.mkdir(config.log_dir) if not os.path.isdir(config.log_dir): os.mkdir(config.log_dir)
if not os.path.isdir(config.data_dir): os.mkdir(config.data_dir) if not os.path.isdir(config.data_dir): os.mkdir(config.data_dir)
if not os.path.isfile("%s/sites.json" % config.data_dir): open("%s/sites.json" % config.data_dir, "w").write("{}") if not os.path.isfile("%s/sites.json" % config.data_dir):
if not os.path.isfile("%s/users.json" % config.data_dir): open("%s/users.json" % config.data_dir, "w").write("{}") open("%s/sites.json" % config.data_dir, "w").write("{}")
if not os.path.isfile("%s/users.json" % config.data_dir):
open("%s/users.json" % config.data_dir, "w").write("{}")
# Setup logging # Setup logging
import logging
if config.action == "main": if config.action == "main":
if os.path.isfile("%s/debug.log" % config.log_dir): # Simple logrotate if os.path.isfile("%s/debug.log" % config.log_dir): # Simple logrotate
if os.path.isfile("%s/debug-last.log" % config.log_dir): os.unlink("%s/debug-last.log" % config.log_dir) if os.path.isfile("%s/debug-last.log" % config.log_dir):
os.unlink("%s/debug-last.log" % config.log_dir)
os.rename("%s/debug.log" % config.log_dir, "%s/debug-last.log" % config.log_dir) os.rename("%s/debug.log" % config.log_dir, "%s/debug-last.log" % config.log_dir)
logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="%s/debug.log" % config.log_dir) logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s',
level=logging.DEBUG, filename="%s/debug.log" % config.log_dir)
else: else:
logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull, "w")) # No file logging if action is not main logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull, "w")) # No file logging if action is not main
@ -39,9 +57,9 @@ if config.debug:
else: else:
console_log.setLevel(logging.INFO) # Display only important info to console console_log.setLevel(logging.INFO) # Display only important info to console
from gevent import monkey; monkey.patch_all(thread=False, ssl=False) # Make time, socket gevent compatible. Not thread: pyfilesystem and system tray icon not compatible, Not ssl: broken in 2.7.9 monkey.patch_all(thread=False, ssl=False) # Make time, socket gevent compatible. Not thread: pyfilesystem and system tray icon not compatible, Not ssl: broken in 2.7.9
import gevent
import time
# Log current config # Log current config
logging.debug("Config: %s" % config) logging.debug("Config: %s" % config)
@ -50,7 +68,7 @@ logging.debug("Config: %s" % config)
# Socks Proxy monkey patch # Socks Proxy monkey patch
if config.proxy: if config.proxy:
from util import SocksProxy from util import SocksProxy
import urllib2
logging.info("Patching sockets to socks proxy: %s" % config.proxy) logging.info("Patching sockets to socks proxy: %s" % config.proxy)
config.fileserver_ip = '127.0.0.1' # Do not accept connections anywhere but localhost config.fileserver_ip = '127.0.0.1' # Do not accept connections anywhere but localhost
SocksProxy.monkeyPath(*config.proxy.split(":")) SocksProxy.monkeyPath(*config.proxy.split(":"))
@ -64,7 +82,7 @@ PluginManager.plugin_manager.loadPlugins()
# -- Actions -- # -- Actions --
@PluginManager.acceptPlugins @PluginManager.acceptPlugins
class Actions: class Actions(object):
# Default action: Start serving UiServer and FileServer # Default action: Start serving UiServer and FileServer
def main(self): def main(self):
logging.info("Version: %s r%s, Python %s, Gevent: %s" % (config.version, config.rev, sys.version, gevent.__version__)) logging.info("Version: %s r%s, Python %s, Gevent: %s" % (config.version, config.rev, sys.version, gevent.__version__))
@ -84,7 +102,6 @@ class Actions:
logging.info("Starting servers....") logging.info("Starting servers....")
gevent.joinall([gevent.spawn(ui_server.start), gevent.spawn(file_server.start)]) gevent.joinall([gevent.spawn(ui_server.start), gevent.spawn(file_server.start)])
# Site commands # Site commands
def siteCreate(self): def siteCreate(self):
@ -115,7 +132,6 @@ class Actions:
logging.info("Site created!") logging.info("Site created!")
def siteSign(self, address, privatekey=None, inner_path="content.json", publish=False): def siteSign(self, address, privatekey=None, inner_path="content.json", publish=False):
from Site import Site from Site import Site
logging.info("Signing site: %s..." % address) logging.info("Signing site: %s..." % address)
@ -128,7 +144,6 @@ class Actions:
if succ and publish: if succ and publish:
self.sitePublish(address, inner_path=inner_path) self.sitePublish(address, inner_path=inner_path)
def siteVerify(self, address): def siteVerify(self, address):
import time import time
from Site import Site from Site import Site
@ -152,7 +167,6 @@ class Actions:
else: else:
logging.error("[ERROR] Error during verifying site files!") logging.error("[ERROR] Error during verifying site files!")
def dbRebuild(self, address): def dbRebuild(self, address):
from Site import Site from Site import Site
logging.info("Rebuilding site sql cache: %s..." % address) logging.info("Rebuilding site sql cache: %s..." % address)
@ -161,7 +175,6 @@ class Actions:
site.storage.rebuildDb() site.storage.rebuildDb()
logging.info("Done in %.3fs" % (time.time()-s)) logging.info("Done in %.3fs" % (time.time()-s))
def dbQuery(self, address, query): def dbQuery(self, address, query):
from Site import Site from Site import Site
import json import json
@ -171,7 +184,6 @@ class Actions:
result.append(dict(row)) result.append(dict(row))
print json.dumps(result, indent=4) print json.dumps(result, indent=4)
def siteAnnounce(self, address): def siteAnnounce(self, address):
from Site.Site import Site from Site.Site import Site
logging.info("Announcing site %s to tracker..." % address) logging.info("Announcing site %s to tracker..." % address)
@ -215,10 +227,7 @@ class Actions:
else: else:
logging.info("No peers found for this site, sitePublish command only works if you already have peers serving your site") logging.info("No peers found for this site, sitePublish command only works if you already have peers serving your site")
# Crypto commands # Crypto commands
def cryptPrivatekeyToAddress(self, privatekey=None): def cryptPrivatekeyToAddress(self, privatekey=None):
from Crypt import CryptBitcoin from Crypt import CryptBitcoin
if not privatekey: # If no privatekey in args then ask it now if not privatekey: # If no privatekey in args then ask it now
@ -227,14 +236,11 @@ class Actions:
print CryptBitcoin.privatekeyToAddress(privatekey) print CryptBitcoin.privatekeyToAddress(privatekey)
def cryptSign(self, message, privatekey): def cryptSign(self, message, privatekey):
from Crypt import CryptBitcoin from Crypt import CryptBitcoin
print CryptBitcoin.sign(message, privatekey) print CryptBitcoin.sign(message, privatekey)
# Peer # Peer
def peerPing(self, peer_ip, peer_port=None): def peerPing(self, peer_ip, peer_port=None):
if not peer_port: if not peer_port:
peer_port = config.fileserver_port peer_port = config.fileserver_port
@ -252,7 +258,6 @@ class Actions:
print "Response time: %.3fs (crypt: %s)" % (time.time()-s, peer.connection.crypt) print "Response time: %.3fs (crypt: %s)" % (time.time()-s, peer.connection.crypt)
time.sleep(1) time.sleep(1)
def peerGetFile(self, peer_ip, peer_port, site, filename): def peerGetFile(self, peer_ip, peer_port, site, filename):
logging.info("Opening a simple connection server") logging.info("Opening a simple connection server")
global file_server global file_server
@ -266,7 +271,6 @@ class Actions:
print peer.getFile(site, filename).read() print peer.getFile(site, filename).read()
print "Response time: %.3fs" % (time.time()-s) print "Response time: %.3fs" % (time.time()-s)
def peerCmd(self, peer_ip, peer_port, cmd, parameters): def peerCmd(self, peer_ip, peer_port, cmd, parameters):
logging.info("Opening a simple connection server") logging.info("Opening a simple connection server")
global file_server global file_server

6
start.py
View file

@ -1,7 +1,13 @@
#!/usr/bin/env python #!/usr/bin/env python
# Included modules
import sys import sys
# ZeroNet Modules
import zeronet import zeronet
def main(): def main():
sys.argv += ["--open_browser", "default_browser"] sys.argv += ["--open_browser", "default_browser"]
zeronet.main() zeronet.main()

17
zeronet.py
View file

@ -1,15 +1,24 @@
#!/usr/bin/env python #!/usr/bin/env python
# Included modules
import os
import gc
import sys
import traceback
# ZeroNet Modules
import update
def main(): def main():
print "- Starting ZeroNet..." print "- Starting ZeroNet..."
import sys, os
main = None main = None
try: try:
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src")) # Imports relative to src sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src")) # Imports relative to src
import main import main
main.start() main.start()
if main.update_after_shutdown: # Updater if main.update_after_shutdown: # Updater
import update, sys, os, gc
# Try cleanup openssl # Try cleanup openssl
try: try:
if "lib.opensslVerify" in sys.modules: if "lib.opensslVerify" in sys.modules:
@ -28,8 +37,7 @@ def main():
handler.close() handler.close()
logger.removeHandler(handler) logger.removeHandler(handler)
except Exception, err: # Prevent closing except (Exception, ): # Prevent closing
import traceback
traceback.print_exc() traceback.print_exc()
traceback.print_exc(file=open("log/error.log", "a")) traceback.print_exc(file=open("log/error.log", "a"))
@ -46,4 +54,3 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
main() main()