diff --git a/plugins/Stats/StatsPlugin.py b/plugins/Stats/StatsPlugin.py index 72c4b7be..06e76cb0 100644 --- a/plugins/Stats/StatsPlugin.py +++ b/plugins/Stats/StatsPlugin.py @@ -1,10 +1,11 @@ import time -import cgi +import html import os import json from Plugin import PluginManager from Config import config +from util import helper @PluginManager.registerTo("UiRequest") @@ -23,7 +24,7 @@ class UiRequestPlugin(object): else: formatted = format % val back.append("
address | request | successive errors | last_request |
---|
address | added | found | latency | successive errors | last_success |
---|
address | connected | peers | content.json | out | in | ||||||
---|---|---|---|---|---|---|---|---|---|---|---|
" % site.address
- for key, peer in site.peers.items():
+ for key, peer in list(site.peers.items()):
if peer.time_found:
time_found = int(time.time() - peer.time_found) / 60
else:
@@ -222,21 +224,21 @@ class UiRequestPlugin(object):
# Big files
yield " Big files: " - for site in self.server.sites.values(): + for site in list(self.server.sites.values()): if not site.settings.get("has_bigfile"): continue bigfiles = {} yield """%s """ % (site.address, site.address) - for peer in site.peers.values(): + for peer in list(site.peers.values()): if not peer.time_piecefields_updated: continue - for sha512, piecefield in peer.piecefields.iteritems(): + for sha512, piecefield in peer.piecefields.items(): if sha512 not in bigfiles: bigfiles[sha512] = [] bigfiles[sha512].append(peer) yield " " % site.address
- for sha512, peers in bigfiles.iteritems():
+ for sha512, peers in bigfiles.items():
yield " - " + sha512 + " (hash id: %s) " % site.content_manager.hashfield.getHashId(sha512) yield "
"
yield " "
@@ -278,12 +280,12 @@ class UiRequestPlugin(object):
yield "Received commands: " yield "
Objects in memory (types: %s, total: %s, %.2fkb): " % ( len(obj_count), - sum([stat[0] for stat in obj_count.values()]), - sum([stat[1] for stat in obj_count.values()]) + sum([stat[0] for stat in list(obj_count.values())]), + sum([stat[1] for stat in list(obj_count.values())]) ) - for obj, stat in sorted(obj_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count - yield " - %.1fkb = %s x %s " % (stat[1], stat[0], obj, cgi.escape(obj)) + for obj, stat in sorted(list(obj_count.items()), key=lambda x: x[1][0], reverse=True): # Sorted by count + yield " - %.1fkb = %s x %s " % (stat[1], stat[0], obj, html.escape(obj)) # Classes @@ -300,70 +302,70 @@ class UiRequestPlugin(object): yield " Classes in memory (types: %s, total: %s, %.2fkb): " % ( len(class_count), - sum([stat[0] for stat in class_count.values()]), - sum([stat[1] for stat in class_count.values()]) + sum([stat[0] for stat in list(class_count.values())]), + sum([stat[1] for stat in list(class_count.values())]) ) - for obj, stat in sorted(class_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count - yield " - %.1fkb = %s x %s " % (stat[1], stat[0], obj, cgi.escape(obj)) + for obj, stat in sorted(list(class_count.items()), key=lambda x: x[1][0], reverse=True): # Sorted by count + yield " - %.1fkb = %s x %s " % (stat[1], stat[0], obj, html.escape(obj)) from greenlet import greenlet objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)] yield " Greenlets (%s): " % len(objs) for obj in objs: - yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj).encode("utf8"))) + yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), html.escape(repr(obj))) from Worker import Worker objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)] yield " Workers (%s): " % len(objs) for obj in objs: - yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) + yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), html.escape(repr(obj))) from Connection import Connection objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)] yield " Connections (%s): " % len(objs) for obj in objs: - yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) + yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), html.escape(repr(obj))) from socket import socket objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)] yield " Sockets (%s): " % len(objs) for obj in objs: - yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) + yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), html.escape(repr(obj))) from msgpack import Unpacker objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)] yield " Msgpack unpacker (%s): " % len(objs) for obj in objs: - yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) + yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), html.escape(repr(obj))) from Site import Site objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)] yield " Sites (%s): " % len(objs) for obj in objs: - yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) + yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), html.escape(repr(obj))) objs = [obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__)] yield " Loggers (%s): " % len(objs) for obj in objs: - yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj.name))) + yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), html.escape(repr(obj.name))) objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)] yield " UiRequests (%s): " % len(objs) for obj in objs: - yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) + yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), html.escape(repr(obj))) from Peer import Peer objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)] yield " Peers (%s): " % len(objs) for obj in objs: - yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) + yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), html.escape(repr(obj))) - objs = [(key, val) for key, val in sys.modules.iteritems() if val is not None] + objs = [(key, val) for key, val in sys.modules.items() if val is not None] objs.sort() yield " Modules (%s): " % len(objs) for module_name, module in objs: - yield " - %.3fkb: %s %s " % (self.getObjSize(module, hpy), module_name, cgi.escape(repr(module))) + yield " - %.3fkb: %s %s " % (self.getObjSize(module, hpy), module_name, html.escape(repr(module))) gc.collect() # Implicit grabage collection yield "Done in %.1f" % (time.time() - s) @@ -398,9 +400,9 @@ class UiRequestPlugin(object): obj_type = str(type(obj)) if obj_type != " " % (attr, cgi.escape(str(getattr(obj, attr)))) + yield "- %s: %s " % (attr, html.escape(str(getattr(obj, attr)))) yield " " gc.collect() # Implicit grabage collection @@ -430,7 +432,7 @@ class UiRequestPlugin(object): """ - yield "Listing all %s objects in memory... " % cgi.escape(type_filter) + yield "Listing all %s objects in memory... " % html.escape(type_filter) ref_count = {} objs = gc.get_objects() @@ -447,7 +449,7 @@ class UiRequestPlugin(object): continue try: yield "%.1fkb %s... " % ( - float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)), cgi.escape(str(obj)[0:100].ljust(100)) + float(sys.getsizeof(obj)) / 1024, html.escape(str(obj)), html.escape(str(obj)[0:100].ljust(100)) ) except: continue @@ -456,7 +458,7 @@ class UiRequestPlugin(object): if "object at" in str(ref) or len(str(ref)) > 100: yield str(ref.__class__.__name__) else: - yield str(ref.__class__.__name__) + ":" + cgi.escape(str(ref)) + yield str(ref.__class__.__name__) + ":" + html.escape(str(ref)) yield "] " ref_type = ref.__class__.__name__ if ref_type not in ref_count: @@ -465,13 +467,14 @@ class UiRequestPlugin(object): ref_count[ref_type][1] += float(sys.getsizeof(obj)) / 1024 # Size yield " " - yield " Object referrer (total: %s, %.2fkb): " % (len(ref_count), sum([stat[1] for stat in ref_count.values()])) + yield " Object referrer (total: %s, %.2fkb): " % (len(ref_count), sum([stat[1] for stat in list(ref_count.values())])) - for obj, stat in sorted(ref_count.items(), key=lambda x: x[1][0], reverse=True)[0:30]: # Sorted by count - yield " - %.1fkb = %s x %s " % (stat[1], stat[0], cgi.escape(str(obj))) + for obj, stat in sorted(list(ref_count.items()), key=lambda x: x[1][0], reverse=True)[0:30]: # Sorted by count + yield " - %.1fkb = %s x %s " % (stat[1], stat[0], html.escape(str(obj))) gc.collect() # Implicit grabage collection + @helper.encodeResponse def actionBenchmark(self): import sys import gc @@ -485,12 +488,14 @@ class UiRequestPlugin(object): @contextmanager def benchmark(name, standard): + self.log.debug("Benchmark: %s" % name) s = time.time() - output("- %s" % name) + output(b"- %s" % name.encode()) try: yield 1 - except Exception, err: - output(" ! Error: %s " % err) + except Exception as err: + self.log.exception(err) + output(b" ! Error: %s " % str(err).encode()) taken = time.time() - s if taken > 0: multipler = standard / taken @@ -512,7 +517,7 @@ class UiRequestPlugin(object): speed = "WOW" else: speed = "Insane!!" - output("%.3fs [x%.2f: %s] " % (taken, multipler, speed)) + output(b"%.3fs [x%.2f: %s] " % (taken, multipler, speed.encode())) time.sleep(0.01) yield """ @@ -550,31 +555,28 @@ class UiRequestPlugin(object): assert sign == valid, "%s != %s" % (sign, valid) address = CryptBitcoin.privatekeyToAddress(privatekey) - if CryptBitcoin.opensslVerify: # Openssl avalible - with benchmark("openssl verify x 100", 0.37): + for lib_verify in ["btctools", "openssl", "libsecp256k1"]: + try: + CryptBitcoin.loadLib(lib_verify) + loaded = True + except Exception as err: + yield "- Error loading %s: %s" % (lib_verify, err) + loaded = False + if not loaded: + continue + with benchmark("%s verify x 100" % lib_verify, 0.37): for i in range(100): if i % 10 == 0: yield "." - ok = CryptBitcoin.verify(data, address, sign) + ok = CryptBitcoin.verify(data, address, sign, lib_verify=lib_verify) assert ok, "does not verify from %s" % address - else: - yield " - openssl verify x 100...not avalible :( " - - openssl_verify_bk = CryptBitcoin.opensslVerify # Emulate openssl not found in any way - CryptBitcoin.opensslVerify = None - with benchmark("pure-python verify x 10", 1.6): - for i in range(10): - yield "." - ok = CryptBitcoin.verify(data, address, sign) - assert ok, "does not verify from %s" % address - CryptBitcoin.opensslVerify = openssl_verify_bk # CryptHash yield " CryptHash: " from Crypt import CryptHash - from cStringIO import StringIO + import io - data = StringIO("Hello" * 1024 * 1024) # 5m + data = io.BytesIO(b"Hello" * 1024 * 1024) # 5m with benchmark("sha256 5M x 10", 0.6): for i in range(10): data.seek(0) @@ -583,7 +585,7 @@ class UiRequestPlugin(object): valid = "8cd629d9d6aff6590da8b80782a5046d2673d5917b99d5603c3dcb4005c45ffa" assert hash == valid, "%s != %s" % (hash, valid) - data = StringIO("Hello" * 1024 * 1024) # 5m + data = io.BytesIO(b"Hello" * 1024 * 1024) # 5m with benchmark("sha512 5M x 10", 0.6): for i in range(10): data.seek(0) @@ -599,34 +601,35 @@ class UiRequestPlugin(object): yield "." # Msgpack - import msgpack - yield " Msgpack: (version: %s) " % ".".join(map(str, msgpack.version)) - binary = 'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv' - data = {"int": 1024*1024*1024, "float": 12345.67890, "text": "hello"*1024, "binary": binary} + from util import Msgpack + yield " Msgpack: (version: %s) " % ".".join(map(str, Msgpack.msgpack.version)) + binary = b'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv' + data = {"int": 1024 * 1024 * 1024, "float": 12345.67890, "text": "hello" * 1024, "binary": binary} with benchmark("pack 5K x 10 000", 0.78): for i in range(10): for y in range(1000): - data_packed = msgpack.packb(data) + data_packed = Msgpack.pack(data) yield "." - valid = """\x84\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohello\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa6binary\xda\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv""" + valid = b"""\x84\xa3int\xce@\x00\x00\x00\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa4text\xda\x14\x00hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohello\xa6binary\xc5\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv""" assert data_packed == valid, "%s != %s" % (repr(data_packed), repr(valid)) with benchmark("unpack 5K x 10 000", 1.2): for i in range(10): for y in range(1000): - data_unpacked = msgpack.unpackb(data_packed) + data_unpacked = Msgpack.unpack(data_packed, decode=False) yield "." assert data == data_unpacked, "%s != %s" % (data_unpacked, data) - with benchmark("streaming unpack 5K x 10 000", 1.4): - for i in range(10): - unpacker = msgpack.Unpacker() - for y in range(1000): - unpacker.feed(data_packed) - for data_unpacked in unpacker: - pass - yield "." - assert data == data_unpacked, "%s != %s" % (data_unpacked, data) + for fallback in [True, False]: + with benchmark("streaming unpack 5K x 10 000 (fallback: %s)" % fallback, 1.4): + for i in range(10): + unpacker = Msgpack.getUnpacker(decode=False, fallback=fallback) + for y in range(1000): + unpacker.feed(data_packed) + for data_unpacked in unpacker: + pass + yield "." + assert data == data_unpacked, "%s != %s" % (data_unpacked, data) # Db from Db import Db @@ -682,7 +685,6 @@ class UiRequestPlugin(object): with benchmark("Buffered insert x 100 x 100", 1.3): cur = db.getCursor() - cur.execute("BEGIN") cur.logging = False for u in range(100, 200): # 100 user data = {"test": []} @@ -693,7 +695,6 @@ class UiRequestPlugin(object): os.unlink("%s/test_%s.json" % (config.data_dir, u)) if u % 10 == 0: yield "." - cur.execute("COMMIT") yield " - Total rows in db: %s " % db.execute("SELECT COUNT(*) AS num FROM test").fetchone()[0] @@ -745,8 +746,8 @@ class UiRequestPlugin(object): # Zip yield " Compression: " import zipfile - test_data = "Test" * 1024 - file_name = "\xc3\x81rv\xc3\xadzt\xc5\xb0r\xc5\x91t\xc3\xbck\xc3\xb6r\xc3\xb3g\xc3\xa9p\xe4\xb8\xad\xe5\x8d\x8e%s.txt" + test_data = b"Test" * 1024 + file_name = b"\xc3\x81rv\xc3\xadzt\xc5\xb0r\xc5\x91t\xc3\xbck\xc3\xb6r\xc3\xb3g\xc3\xa9p\xe4\xb8\xad\xe5\x8d\x8e%s.txt".decode("utf8") with benchmark("Zip pack x 10", 0.12): for i in range(10): @@ -755,88 +756,69 @@ class UiRequestPlugin(object): zip_info = zipfile.ZipInfo(file_name % y, (1980,1,1,0,0,0)) zip_info.compress_type = zipfile.ZIP_DEFLATED zip_info.create_system = 3 + zip_info.flag_bits = 0 archive.writestr(zip_info, test_data) yield "." hash = CryptHash.sha512sum(open("%s/test.zip" % config.data_dir, "rb")) - valid = "f6ef623e6653883a1758db14aa593350e26c9dc53a8406d6e6defd6029dbd483" + valid = "f630fece29fff1cc8dbf454e47a87fea2746a4dbbd2ceec098afebab45301562" assert hash == valid, "Invalid hash: %s != %s " % (hash, valid) with benchmark("Zip unpack x 10", 0.2): for i in range(10): with zipfile.ZipFile('%s/test.zip' % config.data_dir) as archive: for y in range(100): - assert archive.read(file_name % y) == test_data + data = archive.open(file_name % y).read() + assert archive.open(file_name % y).read() == test_data, "Invalid data: %s..." % data[0:30] yield "." if os.path.isfile("%s/test.zip" % config.data_dir): os.unlink("%s/test.zip" % config.data_dir) - # Tar.gz + # gz, bz2, xz import tarfile - import struct + import gzip # Monkey patch _init_write_gz to use fixed date in order to keep the hash independent from datetime def nodate_write_gzip_header(self): - self.mtime = 0 + self._write_mtime = 0 original_write_gzip_header(self) - import gzip original_write_gzip_header = gzip.GzipFile._write_gzip_header gzip.GzipFile._write_gzip_header = nodate_write_gzip_header - test_data_io = StringIO("Test" * 1024) - with benchmark("Tar.gz pack x 10", 0.3): - for i in range(10): - with tarfile.open('%s/test.tar.gz' % config.data_dir, 'w:gz') as archive: - for y in range(100): - test_data_io.seek(0) - tar_info = tarfile.TarInfo(file_name % y) - tar_info.size = 4 * 1024 - archive.addfile(tar_info, test_data_io) - yield "." + test_data_io = io.BytesIO(b"Test" * 1024) + archive_formats = { + "gz": {"hash": "4704ebd8c987ed6f833059f1de9c475d443b0539b8d4c4cb8b49b26f7bbf2d19", "time_pack": 0.3, "time_unpack": 0.2}, + "bz2": {"hash": "90cba0b4d9abaa37b830bf37e4adba93bfd183e095b489ebee62aaa94339f3b5", "time_pack": 2.0, "time_unpack": 0.5}, + "xz": {"hash": "37abc16d552cfd4a495cb2acbf8b1d5877631d084f6571f4d6544bc548c69bae", "time_pack": 1.4, "time_unpack": 0.2} + } + for ext, format_data in archive_formats.items(): + archive_path = '%s/test.tar.%s' % (config.data_dir, ext) + with benchmark("Tar.%s pack x 10" % ext, format_data["time_pack"]): + for i in range(10): + with tarfile.open(archive_path, 'w:%s' % ext) as archive: + for y in range(100): + test_data_io.seek(0) + tar_info = tarfile.TarInfo(file_name % y) + tar_info.size = 4 * 1024 + archive.addfile(tar_info, test_data_io) + yield "." - hash = CryptHash.sha512sum(open("%s/test.tar.gz" % config.data_dir, "rb")) - valid = "4704ebd8c987ed6f833059f1de9c475d443b0539b8d4c4cb8b49b26f7bbf2d19" - assert hash == valid, "Invalid hash: %s != %s " % (hash, valid) + hash = CryptHash.sha512sum(open("%s/test.tar.%s" % (config.data_dir, ext), "rb")) + valid = format_data["hash"] + assert hash == valid, "Invalid hash: %s != %s " % (hash, valid) - with benchmark("Tar.gz unpack x 10", 0.2): - for i in range(10): - with tarfile.open('%s/test.tar.gz' % config.data_dir, 'r:gz') as archive: - for y in range(100): - assert archive.extractfile(file_name % y).read() == test_data - yield "." - - if os.path.isfile("%s/test.tar.gz" % config.data_dir): - os.unlink("%s/test.tar.gz" % config.data_dir) - - # Tar.bz2 - import tarfile - test_data_io = StringIO("Test" * 1024) - with benchmark("Tar.bz2 pack x 10", 2.0): - for i in range(10): - with tarfile.open('%s/test.tar.bz2' % config.data_dir, 'w:bz2') as archive: - for y in range(100): - test_data_io.seek(0) - tar_info = tarfile.TarInfo(file_name % y) - tar_info.size = 4 * 1024 - archive.addfile(tar_info, test_data_io) - yield "." - - hash = CryptHash.sha512sum(open("%s/test.tar.bz2" % config.data_dir, "rb")) - valid = "90cba0b4d9abaa37b830bf37e4adba93bfd183e095b489ebee62aaa94339f3b5" - assert hash == valid, "Invalid hash: %s != %s " % (hash, valid) - - with benchmark("Tar.bz2 unpack x 10", 0.5): - for i in range(10): - with tarfile.open('%s/test.tar.bz2' % config.data_dir, 'r:bz2') as archive: - for y in range(100): - assert archive.extractfile(file_name % y).read() == test_data - yield "." - - if os.path.isfile("%s/test.tar.bz2" % config.data_dir): - os.unlink("%s/test.tar.bz2" % config.data_dir) + archive_size = os.path.getsize(archive_path) / 1024 + with benchmark("Tar.%s unpack (%.2fkB) x 10" % (ext, archive_size), format_data["time_unpack"]): + for i in range(10): + with tarfile.open(archive_path, 'r:%s' % ext) as archive: + for y in range(100): + assert archive.extractfile(file_name % y).read() == test_data + yield "." + if os.path.isfile(archive_path): + os.unlink(archive_path) yield " Done. Total: %.2fs" % (time.time() - t) |