rev119, Protection against update flood, Cache webfonts, Publish batching, Task failed holds Peer objects, Remove peer from failed on addTask, Noparallel memory leak fix

This commit is contained in:
HelloZeroNet 2015-04-24 02:36:00 +02:00
parent f576527986
commit f7717b1de8
14 changed files with 238 additions and 48 deletions

View file

@ -4,7 +4,7 @@ import ConfigParser
class Config(object):
def __init__(self):
self.version = "0.2.9"
self.rev = 116
self.rev = 119
self.parser = self.createArguments()
argv = sys.argv[:] # Copy command line arguments
argv = self.parseConfig(argv) # Add arguments from config file

View file

@ -117,6 +117,9 @@ class Connection:
# Message loop for connection
def messageLoop(self, firstchar=None):
if not self.sock:
self.log("Socket error: No socket found")
return False
sock = self.sock
try:
if not firstchar: firstchar = sock.recv(1)
@ -317,4 +320,5 @@ class Connection:
# Little cleanup
del self.unpacker
del self.sock
self.sock = None
self.unpacker = None

View file

@ -14,6 +14,7 @@ class ConnectionServer:
self.port = port
self.last_connection_id = 1 # Connection id incrementer
self.log = logging.getLogger("ConnServer")
self.port_opened = None
self.connections = [] # Connections
self.ips = {} # Connection by ip

View file

@ -2,6 +2,7 @@ import os, msgpack, shutil, gevent, socket, struct, random
from cStringIO import StringIO
from Debug import Debug
from Config import config
from util import RateLimit
FILE_BUFF = 1024*512
@ -14,6 +15,7 @@ class FileRequest:
self.req_id = None
self.sites = self.server.sites
self.log = server.log
self.responded = False # Responded to the request
def unpackAddress(self, packed):
@ -21,24 +23,34 @@ class FileRequest:
def send(self, msg):
self.connection.send(msg)
if not self.connection.closed:
self.connection.send(msg)
def response(self, msg):
if self.responded:
self.log.debug("Req id %s already responded" % self.req_id)
return
if not isinstance(msg, dict): # If msg not a dict create a {"body": msg}
msg = {"body": msg}
msg["cmd"] = "response"
msg["to"] = self.req_id
self.responded = True
self.send(msg)
# Route file requests
def route(self, cmd, req_id, params):
self.req_id = req_id
if cmd == "getFile":
self.actionGetFile(params)
elif cmd == "update":
self.actionUpdate(params)
event = "%s update %s %s" % (self.connection.id, params["site"], params["inner_path"])
if not RateLimit.isAllowed(event): # There was already an updat for this file in the last 10 second
self.response({"ok": "File update queued"})
RateLimit.callAsync(event, 10, self.actionUpdate, params) # If called more than once within 10 sec only keep the last update
elif cmd == "pex":
self.actionPex(params)
elif cmd == "ping":

View file

@ -96,7 +96,6 @@ class Site:
# Download all file from content.json
@util.Noparallel(blocking=True)
def downloadContent(self, inner_path, download_files=True, peer=None):
s = time.time()
self.log.debug("Downloading %s..." % inner_path)
@ -223,6 +222,7 @@ class Site:
# Update content.json on peers
@util.Noparallel()
def publish(self, limit=5, inner_path="content.json"):
self.log.info( "Publishing to %s/%s peers..." % (limit, len(self.peers)) )
published = [] # Successfully published (Peer)

View file

@ -111,7 +111,7 @@ class UiRequest(object):
if self.env["REQUEST_METHOD"] == "OPTIONS":
headers.append(("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")) # Allow json access
if (self.env["REQUEST_METHOD"] == "OPTIONS" or not self.isAjaxRequest()) and status == 200 and (content_type == "text/css" or content_type == "application/javascript" or self.env["REQUEST_METHOD"] == "OPTIONS" or content_type.startswith("image")): # Cache Css, Js, Image files for 10min
if (self.env["REQUEST_METHOD"] == "OPTIONS" or not self.isAjaxRequest()) and status == 200 and (content_type == "text/css" or content_type.startswith("application") or self.env["REQUEST_METHOD"] == "OPTIONS" or content_type.startswith("image")): # Cache Css, Js, Image files for 10min
headers.append(("Cache-Control", "public, max-age=600")) # Cache 10 min
else: # Images, Css, Js
headers.append(("Cache-Control", "no-cache, no-store, private, must-revalidate, max-age=0")) # No caching at all

View file

@ -2,7 +2,7 @@ import json, gevent, time, sys, hashlib
from Config import config
from Site import SiteManager
from Debug import Debug
from util import QueryJson
from util import QueryJson, RateLimit
from Plugin import PluginManager
@PluginManager.acceptPlugins
@ -149,21 +149,6 @@ class UiWebsocket(object):
func(req["id"], params)
# - Actions -
# Do callback on response {"cmd": "response", "to": message_id, "result": result}
def actionResponse(self, to, result):
if to in self.waiting_cb:
self.waiting_cb[to](result) # Call callback function
else:
self.log.error("Websocket callback not found: %s, %s" % (to, result))
# Send a simple pong answer
def actionPing(self, to):
self.response(to, "pong")
# Format site info
def formatSiteInfo(self, site, create_user=True):
content = site.content_manager.contents.get("content.json")
@ -198,21 +183,6 @@ class UiWebsocket(object):
return ret
# Send site details
def actionSiteInfo(self, to, file_status = None):
ret = self.formatSiteInfo(self.site)
if file_status: # Client queries file status
if self.site.storage.isFile(file_status): # File exits, add event done
ret["event"] = ("file_done", file_status)
self.response(to, ret)
# Join to an event channel
def actionChannelJoin(self, to, channel):
if channel not in self.channels:
self.channels.append(channel)
def formatServerInfo(self):
return {
"ip_external": bool(sys.modules["main"].file_server.port_opened),
@ -228,6 +198,36 @@ class UiWebsocket(object):
}
# - Actions -
# Do callback on response {"cmd": "response", "to": message_id, "result": result}
def actionResponse(self, to, result):
if to in self.waiting_cb:
self.waiting_cb[to](result) # Call callback function
else:
self.log.error("Websocket callback not found: %s, %s" % (to, result))
# Send a simple pong answer
def actionPing(self, to):
self.response(to, "pong")
# Send site details
def actionSiteInfo(self, to, file_status = None):
ret = self.formatSiteInfo(self.site)
if file_status: # Client queries file status
if self.site.storage.isFile(file_status): # File exits, add event done
ret["event"] = ("file_done", file_status)
self.response(to, ret)
# Join to an event channel
def actionChannelJoin(self, to, channel):
if channel not in self.channels:
self.channels.append(channel)
# Server variables
def actionServerInfo(self, to):
ret = self.formatServerInfo()
@ -261,18 +261,28 @@ class UiWebsocket(object):
site.saveSettings()
site.announce()
published = site.publish(5, inner_path) # Publish to 5 peer
event_name = "publish %s %s" % (site.address, inner_path)
thread = RateLimit.callAsync(event_name, 7, site.publish, 5, inner_path) # Only publish once in 7 second to 5 peers
notification = "linked" not in dir(thread) # Only display notification on first callback
thread.linked = True
thread.link(lambda thread: self.cbSitePublish(to, thread, notification)) # At the end callback with request id and thread
# Callback of site publish
def cbSitePublish(self, to, thread, notification=True):
site = self.site
published = thread.value
if published>0: # Successfuly published
self.cmd("notification", ["done", "Content published to %s peers." % published, 5000])
if notification: self.cmd("notification", ["done", "Content published to %s peers." % published, 5000])
self.response(to, "ok")
site.updateWebsocket() # Send updated site data to local websocket clients
if notification: site.updateWebsocket() # Send updated site data to local websocket clients
else:
if len(site.peers) == 0:
self.cmd("notification", ["info", "No peers found, but your content is ready to access."])
if notification: self.cmd("notification", ["info", "No peers found, but your content is ready to access."])
self.response(to, "No peers found, but your content is ready to access.")
else:
self.cmd("notification", ["error", "Content publish failed."])
if notification: self.cmd("notification", ["error", "Content publish failed."])
self.response(to, "Content publish failed.")
@ -326,6 +336,7 @@ class UiWebsocket(object):
return self.response(to, body)
# - Admin actions -
# List all site info

Binary file not shown.

BIN
src/Ui/media/img/logo.psd Normal file

Binary file not shown.

View file

@ -61,7 +61,7 @@ class Worker:
self.task = None
else: # Hash failed
self.manager.log.debug("%s: Hash failed: %s, failed peers: %s" % (self.key, task["inner_path"], len(task["failed"])))
task["failed"].append(self.key)
task["failed"].append(self.peer)
self.task = None
self.peer.hash_failed += 1
if self.peer.hash_failed >= 3: # Broken peer

View file

@ -76,7 +76,7 @@ class WorkerManager:
self.tasks.sort(key=self.taskSorter, reverse=True) # Sort tasks by priority and worker numbers
for task in self.tasks: # Find a task
if task["peers"] and peer not in task["peers"]: continue # This peer not allowed to pick this task
if peer.key in task["failed"]: continue # Peer already tried to solve this, but failed
if peer in task["failed"]: continue # Peer already tried to solve this, but failed
return task
@ -145,6 +145,12 @@ class WorkerManager:
task["peers"].append(peer)
self.log.debug("Added peer %s to %s" % (peer.key, task["inner_path"]))
self.startWorkers([peer])
elif peer and peer in task["failed"]:
task["failed"].remove(peer) # New update arrived, remove the peer from failed peers
self.log.debug("Removed peer %s from failed %s" % (peer.key, task["inner_path"]))
self.startWorkers([peer])
if priority:
task["priority"] += priority # Boost on priority
return task["evt"]

View file

@ -25,9 +25,26 @@ class Event(list):
return self
if __name__ == "__main__":
def testBenchmark():
def say(pre, text):
print "%s Say: %s" % (pre, text)
import time
s = time.time()
onChanged = Event()
for i in range(1000):
onChanged.once(lambda pre: say(pre, "once"), "once")
print "Created 1000 once in %.3fs" % (time.time()-s)
onChanged("#1")
def testUsage():
def say(pre, text):
print "%s Say: %s" % (pre, text)
onChanged = Event()
onChanged.once(lambda pre: say(pre, "once"))
onChanged.once(lambda pre: say(pre, "once"))
@ -37,3 +54,7 @@ if __name__ == "__main__":
onChanged("#1")
onChanged("#2")
onChanged("#3")
if __name__ == "__main__":
testBenchmark()

View file

@ -1,5 +1,6 @@
import gevent, time
class Noparallel(object): # Only allow function running once in same time
def __init__(self,blocking=True):
self.threads = {}
@ -30,15 +31,21 @@ class Noparallel(object): # Only allow function running once in same time
if key in self.threads: del(self.threads[key]) # Allowing it to run again
return ret
else: # No blocking just return the thread
thread.link(lambda thread: self.cleanup(key, thread))
return thread
wrapper.func_name = func.func_name
return wrapper
# Cleanup finished threads
def cleanup(self, key, thread):
if key in self.threads: del(self.threads[key])
class Test():
@Noparallel()
def count(self):
for i in range(5):
def count(self, num=5):
for i in range(num):
print self, i
time.sleep(1)
return "%s return:%s" % (self, i)
@ -46,8 +53,8 @@ class Test():
class TestNoblock():
@Noparallel(blocking=False)
def count(self):
for i in range(5):
def count(self, num=5):
for i in range(num):
print self, i
time.sleep(1)
return "%s return:%s" % (self, i)
@ -104,11 +111,33 @@ def testNoblocking():
print thread1.value, thread2.value, thread3.value, thread4.value
print "Done."
def testBenchmark():
import time
def printThreadNum():
import gc
from greenlet import greenlet
objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
print "Greenlets: %s" % len(objs)
printThreadNum()
test = TestNoblock()
s = time.time()
for i in range(3):
gevent.spawn(test.count, i+1)
print "Created in %.3fs" % (time.time()-s)
printThreadNum()
time.sleep(5)
if __name__ == "__main__":
from gevent import monkey
monkey.patch_all()
testBenchmark()
print "Testing blocking mode..."
testBlocking()
print "Testing noblocking mode..."
testNoblocking()
print [instance.threads for instance in registry]

106
src/util/RateLimit.py Normal file
View file

@ -0,0 +1,106 @@
import time
import gevent
import logging
log = logging.getLogger("RateLimit")
called_db = {}
queue_db = {}
# Register event as called
# Return: None
def called(event):
called_db[event] = time.time()
# Check if calling event is allowed
# Return: True if allowed False if not
def isAllowed(event, allowed_again=10):
last_called = called_db.get(event)
if not last_called: # Its not called before
return True
elif time.time()-last_called >= allowed_again:
del called_db[event] # Delete last call time to save memory
return True
else:
return False
def callQueue(event):
func, args, kwargs, thread = queue_db[event]
log.debug("Calling: %s" % event)
del called_db[event]
del queue_db[event]
return func(*args, **kwargs)
# Rate limit and delay function call if needed, If the function called again within the rate limit interval then previous queued call will be dropped
# Return: Immedietly gevent thread
def callAsync(event, allowed_again=10, func=None, *args, **kwargs):
if isAllowed(event): # Not called recently, call it now
called(event)
# print "Calling now"
return gevent.spawn(func, *args, **kwargs)
else: # Called recently, schedule it for later
time_left = allowed_again-max(0, time.time()-called_db[event])
log.debug("Added to queue (%.2fs left): %s " % (time_left, event))
if not queue_db.get(event): # Function call not queued yet
thread = gevent.spawn_later(time_left, lambda: callQueue(event)) # Call this function later
queue_db[event] = (func, args, kwargs, thread)
return thread
else: # Function call already queued, just update the parameters
thread = queue_db[event][3]
queue_db[event] = (func, args, kwargs, thread)
return thread
# Rate limit and delay function call if needed
# Return: Wait for execution/delay then return value
def call(event, allowed_again=10, func=None, *args, **kwargs):
if isAllowed(event): # Not called recently, call it now
called(event)
# print "Calling now"
return func(*args, **kwargs)
else: # Called recently, schedule it for later
time_left = max(0, allowed_again-(time.time()-called_db[event]))
# print "Time left: %s" % time_left, args, kwargs
log.debug("Calling sync (%.2fs left): %s" % (time_left, event))
time.sleep(time_left)
called(event)
back = func(*args, **kwargs)
if event in called_db:
del called_db[event]
return back
if __name__ == "__main__":
from gevent import monkey
monkey.patch_all()
import random
def publish(inner_path):
print "Publishing %s..." % inner_path
return 1
def cb(thread):
print "Value:", thread.value
print "Testing async spam requests rate limit to 1/sec..."
for i in range(3000):
thread = callAsync("publish content.json", 1, publish, "content.json %s" % i)
time.sleep(float(random.randint(1,20))/100000)
print thread.link(cb)
print "Done"
time.sleep(2)
print "Testing sync spam requests rate limit to 1/sec..."
for i in range(5):
call("publish data.json", 1, publish, "data.json %s" % i)
time.sleep(float(random.randint(1,100))/100)
print "Done"
print called_db, queue_db