more detailed stats, memory optimalizations, connection pinging and timeout, request timeout, validate content after signing, only recompile changed coffeescripts, remove unnecessary js logs

src/Worker/Worker.py

@@ -12,6 +12,14 @@ class Worker:
 		self.thread = None
 
 
+	def __str__(self):
+		return "Worker %s %s" % (self.manager.site.address_short, self.key)
+
+
+	def __repr__(self):
+		return "<%s>" % self.__str__()
+
+
 	# Downloader thread
 	def downloader(self):
 		self.peer.hash_failed = 0 # Reset hash error counter
@@ -34,7 +42,7 @@ class Worker:
 				buff = self.peer.getFile(task["site"].address, task["inner_path"])
 				if self.running == False: # Worker no longer needed or got killed
 					self.manager.log.debug("%s: No longer needed, returning: %s" % (self.key, task["inner_path"]))
-					return None
+					break
 				if buff: # Download ok
 					correct = task["site"].content_manager.verifyFile(task["inner_path"], buff)
 				else: # Download error
@@ -78,4 +86,5 @@ class Worker:
 		self.running = False
 		if self.thread:
 			self.thread.kill(exception=Debug.Notify("Worker stopped"))
+		del self.thread
 		self.manager.removeWorker(self)

src/Worker/WorkerManager.py

@@ -8,16 +8,26 @@ class WorkerManager:
 	def __init__(self, site):
 		self.site = site
 		self.workers = {} # Key: ip:port, Value: Worker.Worker
-		self.tasks = [] # {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_started": None, "time_added": time.time(), "peers": peers, "priority": 0}
+		self.tasks = [] # {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_started": None, "time_added": time.time(), "peers": peers, "priority": 0, "failed": peer_ids}
 		self.started_task_num = 0 # Last added task num
 		self.running = True
 		self.log = logging.getLogger("WorkerManager:%s" % self.site.address_short)
 		self.process_taskchecker = gevent.spawn(self.checkTasks)
 
 
+	def __str__(self):
+		return "WorkerManager %s" % self.site.address_short
+
+
+	def __repr__(self):
+		return "<%s>" % self.__str__()
+
+
+
 	# Check expired tasks
 	def checkTasks(self):
 		while self.running:
+			tasks = task = worker = workers = None # Cleanup local variables
 			time.sleep(15) # Check every 15 sec
 
 			# Clean up workers
@@ -25,6 +35,7 @@ class WorkerManager:
 				if worker.task and worker.task["done"]: worker.stop() # Stop workers with task done
 
 			if not self.tasks: continue
+
 			tasks = self.tasks[:] # Copy it so removing elements wont cause any problem
 			for task in tasks:
 				if (task["time_started"] and time.time() >= task["time_started"]+60) or (time.time() >= task["time_added"]+60 and not self.workers): # Task taking too long time, or no peer after 60sec kill it
@@ -44,6 +55,8 @@ class WorkerManager:
 							task["peers"] = []
 							self.startWorkers()
 						break # One reannounce per loop
+
+
 		self.log.debug("checkTasks stopped running")