rev119, Protection against update flood, Cache webfonts, Publish batching, Task failed holds Peer objects, Remove peer from failed on addTask, Noparallel memory leak fix

src/util/Event.py

@@ -25,9 +25,26 @@ class Event(list):
 		return self
 
 
-if __name__ == "__main__":
+
+
+def testBenchmark():
 	def say(pre, text):
 		print "%s Say: %s" % (pre, text)
+	
+	import time
+	s = time.time()
+	onChanged = Event()
+	for i in range(1000):
+		onChanged.once(lambda pre: say(pre, "once"), "once")
+	print "Created 1000 once in %.3fs" % (time.time()-s)
+	onChanged("#1")
+
+
+
+def testUsage():
+	def say(pre, text):
+		print "%s Say: %s" % (pre, text)
+	
 	onChanged = Event()
 	onChanged.once(lambda pre: say(pre, "once"))
 	onChanged.once(lambda pre: say(pre, "once"))
@@ -37,3 +54,7 @@ if __name__ == "__main__":
 	onChanged("#1")
 	onChanged("#2")
 	onChanged("#3")
+
+
+if __name__ == "__main__":
+	testBenchmark()

src/util/Noparallel.py

@@ -1,5 +1,6 @@
 import gevent, time
 
+
 class Noparallel(object): # Only allow function running once in same time
 	def __init__(self,blocking=True):
 		self.threads = {}
@@ -30,15 +31,21 @@ class Noparallel(object): # Only allow function running once in same time
 					if key in self.threads: del(self.threads[key]) # Allowing it to run again
 					return ret
 				else: # No blocking just return the thread
+					thread.link(lambda thread: self.cleanup(key, thread))
 					return thread
 		wrapper.func_name = func.func_name
 		
 		return wrapper
 
+	# Cleanup finished threads
+	def cleanup(self, key, thread):
+		if key in self.threads: del(self.threads[key])
+
+
 class Test():
 	@Noparallel()
-	def count(self):
-		for i in range(5):
+	def count(self, num=5):
+		for i in range(num):
 			print self, i
 			time.sleep(1)
 		return "%s return:%s" % (self, i)
@@ -46,8 +53,8 @@ class Test():
 
 class TestNoblock():
 	@Noparallel(blocking=False)
-	def count(self):
-		for i in range(5):
+	def count(self, num=5):
+		for i in range(num):
 			print self, i
 			time.sleep(1)
 		return "%s return:%s" % (self, i)
@@ -104,11 +111,33 @@ def testNoblocking():
 	print thread1.value, thread2.value, thread3.value, thread4.value
 	print "Done."
 
+
+def testBenchmark():
+	import time
+	def printThreadNum():
+		import gc
+		from greenlet import greenlet
+		objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
+		print "Greenlets: %s" % len(objs)
+
+	printThreadNum()
+	test = TestNoblock()
+	s = time.time()
+	for i in range(3):
+		gevent.spawn(test.count, i+1)
+	print "Created in %.3fs" % (time.time()-s)
+	printThreadNum()
+	time.sleep(5)
+
+
+
 if __name__ == "__main__":
 	from gevent import monkey
 	monkey.patch_all()
 
+	testBenchmark()
 	print "Testing blocking mode..."
 	testBlocking()
 	print "Testing noblocking mode..."
 	testNoblocking()
+	print [instance.threads for instance in registry]

src/util/RateLimit.py

@@ -0,0 +1,106 @@
+import time
+import gevent
+import logging
+
+log = logging.getLogger("RateLimit")
+
+called_db = {}
+queue_db = {}
+
+# Register event as called
+# Return: None
+def called(event):
+	called_db[event] = time.time()
+
+
+# Check if calling event is allowed
+# Return: True if allowed False if not
+def isAllowed(event, allowed_again=10):
+	last_called = called_db.get(event)
+	if not last_called: # Its not called before
+		return True
+	elif time.time()-last_called >= allowed_again:
+		del called_db[event] # Delete last call time to save memory
+		return True
+	else:
+		return False
+
+
+def callQueue(event):
+	func, args, kwargs, thread = queue_db[event]
+	log.debug("Calling: %s" % event)
+	del called_db[event]
+	del queue_db[event]
+	return func(*args, **kwargs)
+
+
+
+# Rate limit and delay function call if needed, If the function called again within the rate limit interval then previous queued call will be dropped
+# Return: Immedietly gevent thread
+def callAsync(event, allowed_again=10, func=None, *args, **kwargs):
+	if isAllowed(event): # Not called recently, call it now
+		called(event)
+		# print "Calling now"
+		return gevent.spawn(func, *args, **kwargs)
+	else: # Called recently, schedule it for later
+		time_left = allowed_again-max(0, time.time()-called_db[event])
+		log.debug("Added to queue (%.2fs left): %s " % (time_left, event))
+		if not queue_db.get(event): # Function call not queued yet
+			thread = gevent.spawn_later(time_left, lambda: callQueue(event)) # Call this function later
+			queue_db[event] = (func, args, kwargs, thread)
+			return thread
+		else: # Function call already queued, just update the parameters
+			thread = queue_db[event][3]
+			queue_db[event] = (func, args, kwargs, thread)
+			return thread
+
+
+# Rate limit and delay function call if needed
+# Return: Wait for execution/delay then return value
+def call(event, allowed_again=10, func=None, *args, **kwargs):
+	if isAllowed(event): # Not called recently, call it now
+		called(event)
+		# print "Calling now"
+		return func(*args, **kwargs)
+
+	else: # Called recently, schedule it for later
+		time_left = max(0, allowed_again-(time.time()-called_db[event]))
+		# print "Time left: %s" % time_left, args, kwargs
+		log.debug("Calling sync (%.2fs left): %s" % (time_left, event))
+		time.sleep(time_left)
+		called(event)
+		back = func(*args, **kwargs)
+		if event in called_db:
+			del called_db[event]
+		return back
+
+
+
+if __name__ == "__main__":
+	from gevent import monkey
+	monkey.patch_all()
+	import random
+
+	def publish(inner_path):
+		print "Publishing %s..." % inner_path
+		return 1
+
+	def cb(thread):
+		print "Value:", thread.value
+
+	print "Testing async spam requests rate limit to 1/sec..."
+	for i in range(3000):
+		thread = callAsync("publish content.json", 1, publish, "content.json %s" % i)
+		time.sleep(float(random.randint(1,20))/100000)
+	print thread.link(cb)
+	print "Done"
+
+	time.sleep(2)
+
+	print "Testing sync spam requests rate limit to 1/sec..."
+	for i in range(5):
+		call("publish data.json", 1, publish, "data.json %s" % i)
+		time.sleep(float(random.randint(1,100))/100)
+	print "Done"
+
+	print called_db, queue_db