version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix

src/File/FileRequest.py

@@ -1,5 +1,4 @@
 import os, msgpack, shutil, gevent
-from Site import SiteManager
 from cStringIO import StringIO
 from Debug import Debug
 from Config import config
@@ -8,21 +7,30 @@ FILE_BUFF = 1024*512
 
 # Request from me
 class FileRequest:
-	def __init__(self, server = None):
-		if server:
-			self.server = server
-			self.log = server.log
-		self.sites = SiteManager.list()
+	def __init__(self, server, connection):
+		self.server = server
+		self.connection = connection
+
+		self.req_id = None
+		self.sites = self.server.sites
+		self.log = server.log
 
 
 	def send(self, msg):
+		self.connection.send(msg)
+
+
+	def response(self, msg):
 		if not isinstance(msg, dict): # If msg not a dict create a {"body": msg}
 			msg = {"body": msg}
-		self.server.socket.send(msgpack.packb(msg, use_bin_type=True))
+		msg["cmd"] = "response"
+		msg["to"] = self.req_id
+		self.send(msg)
 
 
 	# Route file requests
-	def route(self, cmd, params):
+	def route(self, cmd, req_id, params):
+		self.req_id = req_id
 		if cmd == "getFile":
 			self.actionGetFile(params)
 		elif cmd == "update":
@@ -37,7 +45,7 @@ class FileRequest:
 	def actionUpdate(self, params):
 		site = self.sites.get(params["site"])
 		if not site or not site.settings["serving"]: # Site unknown or not serving
-			self.send({"error": "Unknown site"})
+			self.response({"error": "Unknown site"})
 			return False
 		if site.settings["own"] and params["inner_path"].endswith("content.json"):
 			self.log.debug("Someone trying to push a file to own site %s, reload local %s first" % (site.address, params["inner_path"]))
@@ -61,7 +69,7 @@ class FileRequest:
 					lambda: site.downloadContent(params["inner_path"], peer=peer)
 				) # Load new content file and download changed files in new thread
 
-			self.send({"ok": "Thanks, file %s updated!" % params["inner_path"]})
+			self.response({"ok": "Thanks, file %s updated!" % params["inner_path"]})
 
 		elif valid == None: # Not changed
 			peer = site.addPeer(*params["peer"], return_peer = True) # Add or get peer
@@ -70,18 +78,18 @@ class FileRequest:
 				for task in site.worker_manager.tasks: # New peer add to every ongoing task
 					if task["peers"]: site.needFile(task["inner_path"], peer=peer, update=True, blocking=False) # Download file from this peer too if its peer locked
 
-			self.send({"ok": "File not changed"})
+			self.response({"ok": "File not changed"})
 
 		else: # Invalid sign or sha1 hash
 			self.log.debug("Update for %s is invalid" % params["inner_path"])
-			self.send({"error": "File invalid"})
+			self.response({"error": "File invalid"})
 
 
 	# Send file content request
 	def actionGetFile(self, params):
 		site = self.sites.get(params["site"])
 		if not site or not site.settings["serving"]: # Site unknown or not serving
-			self.send({"error": "Unknown site"})
+			self.response({"error": "Unknown site"})
 			return False
 		try:
 			file_path = site.getPath(params["inner_path"])
@@ -93,18 +101,18 @@ class FileRequest:
 			back["location"] = file.tell()
 			back["size"] = os.fstat(file.fileno()).st_size
 			if config.debug_socket: self.log.debug("Sending file %s from position %s to %s" % (file_path, params["location"], back["location"]))
-			self.send(back)
+			self.response(back)
 			if config.debug_socket: self.log.debug("File %s sent" % file_path)
 		except Exception, err:
-			self.send({"error": "File read error: %s" % Debug.formatException(err)})
+			self.response({"error": "File read error: %s" % Debug.formatException(err)})
 			return False
 
 
 	# Send a simple Pong! answer
 	def actionPing(self):
-		self.send("Pong!")
+		self.response("Pong!")
 
 
 	# Unknown command
 	def actionUnknown(self, cmd, params):
-		self.send({"error": "Unknown command: %s" % cmd})
+		self.response({"error": "Unknown command: %s" % cmd})

src/File/FileServer.py

@@ -5,30 +5,28 @@ from Config import config
 from FileRequest import FileRequest
 from Site import SiteManager
 from Debug import Debug
+from Connection import ConnectionServer
 
 
-class FileServer:
+class FileServer(ConnectionServer):
 	def __init__(self):
-		self.ip = config.fileserver_ip
-		self.port = config.fileserver_port
-		self.log = logging.getLogger(__name__)
+		ConnectionServer.__init__(self, config.fileserver_ip, config.fileserver_port, self.handleRequest)
 		if config.ip_external: # Ip external definied in arguments
 			self.port_opened = True
 			SiteManager.peer_blacklist.append((config.ip_external, self.port)) # Add myself to peer blacklist
 		else:
 			self.port_opened = None # Is file server opened on router
 		self.sites = SiteManager.list()
-		self.running = True
 
 
 	# Handle request to fileserver
-	def handleRequest(self, msg):
-		if "params" in msg:
-			self.log.debug("FileRequest: %s %s %s" % (msg["cmd"], msg["params"].get("site"), msg["params"].get("inner_path")))
+	def handleRequest(self, connection, message):
+		if "params" in message:
+			self.log.debug("FileRequest: %s %s %s" % (message["cmd"], message["params"].get("site"), message["params"].get("inner_path")))
 		else:
-			self.log.debug("FileRequest: %s" % msg["cmd"])
-		req = FileRequest(self)
-		req.route(msg["cmd"], msg.get("params"))
+			self.log.debug("FileRequest: %s" % req["cmd"])
+		req = FileRequest(self, connection)
+		req.route(message["cmd"], message.get("req_id"), message.get("params"))
 
 
 	# Reload the FileRequest class to prevent restarts in debug mode
@@ -124,13 +122,15 @@ class FileServer:
 			time.sleep(2) # Prevent too quick request
 
 
-	# Announce sites every 10 min
+	# Announce sites every 20 min
 	def announceSites(self):
 		while 1:
 			time.sleep(20*60) # Announce sites every 20 min
 			for address, site in self.sites.items():
 				if site.settings["serving"]:
 					site.announce() # Announce site to tracker
+					for inner_path in site.bad_files: # Reset bad file retry counter
+						site.bad_files[inner_path] = 0
 				time.sleep(2) # Prevent too quick request
 
 
@@ -155,40 +155,14 @@ class FileServer:
 			from Debug import DebugReloader
 			DebugReloader(self.reload)
 
-		self.context = zmq.Context()
-		socket = self.context.socket(zmq.REP)
-		self.socket = socket
-		self.socket.setsockopt(zmq.RCVTIMEO, 5000) # Wait for data receive
-		self.socket.setsockopt(zmq.SNDTIMEO, 50000) # Wait for data send
-		self.log.info("Binding to tcp://%s:%s" % (self.ip, self.port))
-		try:
-			self.socket.bind('tcp://%s:%s' % (self.ip, self.port))
-		except Exception, err:
-			self.log.error("Can't bind, FileServer must be running already")
-			return
 		if check_sites: # Open port, Update sites, Check files integrity
 			gevent.spawn(self.checkSites)
 		
 		thread_announce_sites = gevent.spawn(self.announceSites)
 		thread_wakeup_watcher = gevent.spawn(self.wakeupWatcher)
 
-		while self.running:
-			try:
-				ret = {}
-				req = msgpack.unpackb(socket.recv())
-				self.handleRequest(req)
-			except Exception, err:
-				self.log.error(err)
-				if self.running: self.socket.send(msgpack.packb({"error": "%s" % Debug.formatException(err)}, use_bin_type=True))
-				if config.debug: # Raise exception
-					import sys
-					sys.modules["src.main"].DebugHook.handleError() 
-		thread_wakeup_watcher.kill(exception=Debug.Notify("Stopping FileServer"))
-		thread_announce_sites.kill(exception=Debug.Notify("Stopping FileServer"))
+		ConnectionServer.start(self)
+
+		# thread_wakeup_watcher.kill(exception=Debug.Notify("Stopping FileServer"))
+		# thread_announce_sites.kill(exception=Debug.Notify("Stopping FileServer"))
 		self.log.debug("Stopped.")
-
-
-	def stop(self):
-		self.running = False
-		self.socket.close()
-