rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer

plugins/Stats/StatsPlugin.py

@@ -119,7 +119,7 @@ class UiRequestPlugin(object):
 		yield "</table>"
 
 
-		# Objects
+		# Object types
 
 		obj_count = {}
 		for obj in gc.get_objects():
@@ -135,6 +135,24 @@ class UiRequestPlugin(object):
 			yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % (stat[1], stat[0], obj, cgi.escape(obj))
 
 
+		# Classes
+
+		class_count = {}
+		for obj in gc.get_objects():
+			obj_type = str(type(obj))
+			if obj_type != "<type 'instance'>": continue
+			class_name = obj.__class__.__name__
+			if not class_name in class_count:
+				class_count[class_name] = [0, 0]
+			class_count[class_name][0] += 1 # Count
+			class_count[class_name][1] += float(sys.getsizeof(obj))/1024 # Size
+
+		yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % (len(class_count), sum([stat[0] for stat in class_count.values()]), sum([stat[1] for stat in class_count.values()]))
+
+		for obj, stat in sorted(class_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count
+			yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % (stat[1], stat[0], obj, cgi.escape(obj))
+
+
 		from greenlet import greenlet
 		objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
 		yield "<br>Greenlets (%s):<br>" % len(objs)
@@ -203,6 +221,29 @@ class UiRequestPlugin(object):
 		yield "Done in %.1f" % (time.time()-s)
 
 
+	def actionDumpobj(self):
+		import gc, sys
+
+		self.sendHeader()
+		class_filter = self.get.get("class")
+
+		yield """
+		<style>
+		 * { font-family: monospace; white-space: pre }
+		 table * { text-align: right; padding: 0px 10px }
+		</style>
+		"""
+
+		objs = gc.get_objects()
+		for obj in objs:
+			obj_type = str(type(obj))
+			if obj_type != "<type 'instance'>" or obj.__class__.__name__ != class_filter: continue
+			yield "%.1fkb %s... " % (float(sys.getsizeof(obj))/1024, cgi.escape(str(obj)) )
+			for attr in dir(obj):
+				yield "- %s: %s<br>" % (attr, cgi.escape(str(getattr(obj, attr))))
+			yield "<br>"
+
+
 	def actionListobj(self):
 		import gc, sys
 

src/Config.py

@@ -4,7 +4,7 @@ import ConfigParser
 class Config(object):
 	def __init__(self):
 		self.version = "0.2.9"
-		self.rev = 122
+		self.rev = 125
 		self.parser = self.createArguments()
 		argv = sys.argv[:] # Copy command line arguments
 		argv = self.parseConfig(argv) # Add arguments from config file
@@ -16,6 +16,11 @@ class Config(object):
 		return str(self.arguments).replace("Namespace", "Config") # Using argparse str output
 
 
+	# Convert string to bool
+	def strToBool(self, v):
+		return v.lower() in ("yes", "true", "t", "1")
+
+
 	# Create command line arguments
 	def createArguments(self):
 		# Platform specific
@@ -23,9 +28,14 @@ class Config(object):
 			coffeescript = "type %s | tools\\coffee\\coffee.cmd"
 		else:
 			coffeescript = None
+		if sys.platform.startswith("Darwin"): # For some reasons openssl doesnt works on mac yet (https://github.com/HelloZeroNet/ZeroNet/issues/94)
+			disable_openssl = True
+		else:
+			disable_openssl = False
 
 		# Create parser
 		parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+		parser.register('type','bool', self.strToBool)
 		subparsers = parser.add_subparsers(title="Action to perform", dest="action")
 
 		# Main
@@ -72,6 +82,13 @@ class Config(object):
 		action.add_argument('site', 			help='Site address')
 		action.add_argument('filename', 		help='File name to request')
 
+		# PeerGetFile
+		action = subparsers.add_parser("peerCmd", help='Request and print a file content from peer')
+		action.add_argument('peer_ip', 			help='Peer ip')
+		action.add_argument('peer_port', 		help='Peer port')
+		action.add_argument('cmd', 				help='Command to execute')
+		action.add_argument('parameters', 		help='Parameters to command', nargs='?')
+
 
 
 		# Config parameters
@@ -88,8 +105,9 @@ class Config(object):
 		parser.add_argument('--fileserver_ip', 	help='FileServer bind address', default="*", metavar='ip')
 		parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port')
 		parser.add_argument('--disable_zeromq', help='Disable compatibility with old clients', action='store_true')
-		parser.add_argument('--proxy',			help='Socks proxy address', metavar='ip:port')
+		parser.add_argument('--disable_openssl',help='Disable usage of OpenSSL liblary', type='bool', choices=[True, False], default=disable_openssl)
 		parser.add_argument('--disable_udp',	help='Disable UDP connections', action='store_true')
+		parser.add_argument('--proxy',			help='Socks proxy address', metavar='ip:port')
 		parser.add_argument('--ip_external',	help='External ip (tested on start if None)', metavar='ip')
 
 		parser.add_argument('--coffeescript_compiler',	help='Coffeescript compiler for developing', default=coffeescript, metavar='executable_path')

src/Connection/Connection.py

@@ -12,7 +12,9 @@ if not config.disable_zeromq:
 
 
 
-class Connection:
+class Connection(object):
+	__slots__ = ("sock", "ip", "port", "peer_id", "id", "protocol", "type", "server", "unpacker", "req_id", "handshake", "connected", "event_connected", "closed", "zmq_sock", "zmq_queue", "zmq_working", "forward_thread", "start_time", "last_recv_time", "last_message_time", "last_send_time", "last_sent_time", "incomplete_buff_recv", "bytes_recv", "bytes_sent", "last_ping_delay", "last_req_time", "last_cmd", "name", "updateName", "waiting_requests")
+
 	def __init__(self, server, ip, port, sock=None):
 		self.sock = sock
 		self.ip = ip
@@ -315,7 +317,7 @@ class Connection:
 				self.sock.shutdown(gevent.socket.SHUT_WR)
 				self.sock.close()
 		except Exception, err:
-			if config.debug_socket: self.log("Close error: %s" % Debug.formatException(err))
+			if config.debug_socket: self.log("Close error: %s" % err)
 
 		# Little cleanup
 		del self.unpacker

src/Content/ContentManager.py

@@ -70,7 +70,7 @@ class ContentManager:
 				self.site.bad_files[inner_path] = True
 
 		if new_content["modified"] > self.site.settings.get("modified", 0):
-			self.site.settings["modified"] = new_content["modified"]
+			self.site.settings["modified"] = min(time.time()+60*10, new_content["modified"]) # Dont store modifications in the far future (more than 10 minute)
 
 		return changed
 

src/Crypt/CryptBitcoin.py

@@ -1,8 +1,11 @@
 from lib.BitcoinECC import BitcoinECC
 from lib.pybitcointools import bitcoin as btctools
 import logging
+from Config import config
+
 # Try to load openssl
 try:
+	if config.disable_openssl: raise Exception("Disabled by config")
 	from lib.opensslVerify import opensslVerify
 	logging.info("OpenSSL loaded, version: %s" % opensslVerify.openssl_version)
 except Exception, err:

src/File/FileRequest.py

@@ -7,7 +7,9 @@ from util import RateLimit
 FILE_BUFF = 1024*512
 
 # Request from me
-class FileRequest:
+class FileRequest(object):
+	__slots__ = ("server", "connection", "req_id", "sites", "log", "responded")
+
 	def __init__(self, server, connection):
 		self.server = server
 		self.connection = connection
@@ -53,6 +55,8 @@ class FileRequest:
 
 		elif cmd == "pex":
 			self.actionPex(params)
+		elif cmd == "modified":
+			self.actionModified(params)
 		elif cmd == "ping":
 			self.actionPing()
 		else:
@@ -158,6 +162,17 @@ class FileRequest:
 		self.response({"peers": packed_peers})
 
 
+	# Get modified content.json files since
+	def actionModified(self, params):
+		site = self.sites.get(params["site"])
+		if not site or not site.settings["serving"]: # Site unknown or not serving
+			self.response({"error": "Unknown site"})
+			return False
+		modified_files = {inner_path: content["modified"] for inner_path, content in site.content_manager.contents.iteritems() if content["modified"] > params["since"]}
+		self.response({"modified_files": modified_files})
+
+
+
 	# Send a simple Pong! answer
 	def actionPing(self):
 		self.response("Pong!")

src/Peer/Peer.py

@@ -4,7 +4,9 @@ from Config import config
 from Debug import Debug
 
 # Communicate remote peers
-class Peer:
+class Peer(object):
+	__slots__ = ("ip", "port", "site", "key", "connection_server", "connection", "last_found", "last_response", "last_ping", "added", "connection_error", "hash_failed", "download_bytes", "download_time")
+
 	def __init__(self, ip, port, site=None):
 		self.ip = ip
 		self.port = port

src/Site/Site.py

@@ -328,7 +328,7 @@ class Site:
 
 	# Add myself and get other peers from tracker
 	def announce(self, force=False):
-		if time.time() < self.last_announce+60 and not force: return # No reannouncing within 60 secs
+		if time.time() < self.last_announce+30 and not force: return # No reannouncing within 30 secs
 		self.last_announce = time.time()
 		errors = []
 		address_hash = hashlib.sha1(self.address).hexdigest()

src/Site/SiteStorage.py

@@ -36,6 +36,7 @@ class SiteStorage:
 
 	def closeDb(self):
 		if self.db: self.db.close()
+		self.db = None
 
 
 	# Return db class
@@ -120,13 +121,17 @@ class SiteStorage:
 	# Write content to file
 	def write(self, inner_path, content):
 		file_path = self.getPath(inner_path)
+		# Create dir if not exits
+		file_dir = os.path.dirname(file_path)
+		if not os.path.isdir(file_dir):
+			os.makedirs(file_dir)
 		# Write file
 		if hasattr(content, 'read'): # File-like object
-			file = open(file_path, "wb")
-			shutil.copyfileobj(content, file) # Write buff to disk
-			file.close()
+			with open(file_path, "wb") as file:
+				shutil.copyfileobj(content, file) # Write buff to disk
 		else: # Simple string
-			open(file_path, "wb").write(content)
+			with open(file_path, "wb") as file:
+				file.write(content)
 		del content
 		self.onUpdated(inner_path)
 
@@ -146,7 +151,8 @@ class SiteStorage:
 
 	# Load and parse json file
 	def loadJson(self, inner_path):
-		return json.load(self.open(inner_path))
+		with self.open(inner_path) as file:
+			return json.load(file)
 
 
 	# Get file size
@@ -164,7 +170,7 @@ class SiteStorage:
 		return os.path.isdir(self.getPath(inner_path))
 
 
-	# Sercurity check and return path of site's file
+	# Security check and return path of site's file
 	def getPath(self, inner_path):
 		inner_path = inner_path.replace("\\", "/") # Windows separator fix
 		inner_path = re.sub("^%s/" % re.escape(self.directory), "", inner_path) # Remove site directory if begins with it
@@ -176,8 +182,6 @@ class SiteStorage:
 
 
 
-
-
 	# Verify all files sha512sum using content.json
 	def verifyFiles(self, quick_check=False): # Fast = using file size
 		bad_files = []

src/Ui/UiServer.py

@@ -40,7 +40,8 @@ class UiWSGIHandler(WSGIHandler):
 					import sys
 					del self.server.sockets[self.client_address]
 					sys.modules["main"].DebugHook.handleError()
-		del self.server.sockets[self.client_address]
+		if self.client_address in self.server.sockets:
+			del self.server.sockets[self.client_address]
 
 
 class UiServer:

src/Worker/Worker.py

@@ -53,8 +53,6 @@ class Worker:
 					if correct == True and task["done"] == False: # Save if changed and task not done yet
 						buff.seek(0)
 						file_path = site.storage.getPath(task["inner_path"])
-						file_dir = os.path.dirname(file_path)
-						if not os.path.isdir(file_dir): os.makedirs(file_dir) # Make directory for files
 						site.storage.write(task["inner_path"], buff)
 					if task["done"] == False: self.manager.doneTask(task)
 					task["workers_num"] -= 1
@@ -81,7 +79,7 @@ class Worker:
 
 	# Force stop the worker
 	def stop(self):
-		self.manager.log.debug("%s: Force stopping, thread: %s" % (self.key, self.thread))
+		self.manager.log.debug("%s: Force stopping, thread" % self.key)
 		self.running = False
 		if self.thread:
 			self.thread.kill(exception=Debug.Notify("Worker stopped"))

src/main.py

@@ -248,6 +248,24 @@ class Actions:
 		print peer.getFile(site, filename).read()
 		print "Response time: %.3fs" % (time.time()-s)
 
+
+	def peerCmd(self, peer_ip, peer_port, cmd, parameters):
+		logging.info("Opening a simple connection server")
+		global file_server
+		from Connection import ConnectionServer
+		file_server = ConnectionServer()
+		from Peer import Peer
+		peer = Peer(peer_ip, peer_port)
+
+		import json
+		if parameters:
+			parameters = json.loads(parameters.replace("'", '"'))
+		else:
+			parameters = {}
+		logging.info("Response: %s" % peer.request(cmd, parameters))
+
+
+
 actions = Actions()
 # Starts here when running zeronet.py
 def start():

zeronet.py

@@ -3,6 +3,7 @@
 def main():
 	print "- Starting ZeroNet..."
 	import sys, os
+	main = None
 	try:
 		sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src")) # Imports relative to src
 		import main
@@ -25,7 +26,7 @@ def main():
 		traceback.print_exc()
 		raw_input("-- Error happened, press enter to close --")
 
-	if main.update_after_shutdown: # Updater
+	if main and main.update_after_shutdown: # Updater
 		# Restart
 		gc.collect() # Garbage collect
 		print "Restarting..."