From 8f63e4c421ca7d9ba2ce1518c0beaade287a64f7 Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Sun, 13 Sep 2015 23:17:13 +0200 Subject: [PATCH] Rev399, Urandom and Msgpack benchmark, Better random string generation, Never render page on OPTIONS request, Fix for Chrome browser socket hang on zeronet version update --- plugins/Stats/StatsPlugin.py | 41 +++++++++++++++++++++++++++++- src/Config.py | 2 +- src/Connection/ConnectionServer.py | 6 ++--- src/Crypt/CryptHash.py | 11 ++++++++ src/Site/Site.py | 9 +++---- src/Ui/UiRequest.py | 9 ++++--- src/Ui/UiServer.py | 24 +++++++++-------- src/Ui/media/Wrapper.css | 1 + src/Ui/media/all.css | 1 + 9 files changed, 79 insertions(+), 25 deletions(-) diff --git a/plugins/Stats/StatsPlugin.py b/plugins/Stats/StatsPlugin.py index 43edefdd..09ba2ee4 100644 --- a/plugins/Stats/StatsPlugin.py +++ b/plugins/Stats/StatsPlugin.py @@ -375,6 +375,7 @@ class UiRequestPlugin(object): t = time.time() + # CryptBitcoin yield "
CryptBitcoin:
" from Crypt import CryptBitcoin @@ -417,12 +418,13 @@ class UiRequestPlugin(object): assert ok, "does not verify from %s" % address CryptBitcoin.opensslVerify = openssl_verify_bk + # CryptHash yield "
CryptHash:
" from Crypt import CryptHash from cStringIO import StringIO data = StringIO("Hello" * 1024 * 1024) # 5m - with benchmark("sha512 x 10 000", 1): + with benchmark("sha512 x 100 000", 1): for i in range(10): for y in range(10000): hash = CryptHash.sha512sum(data) @@ -430,6 +432,43 @@ class UiRequestPlugin(object): valid = "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce" assert hash == valid, "%s != %s" % (hash, valid) + with benchmark("os.urandom(256) x 100 000", 0.65): + for i in range(10): + for y in range(10000): + data = os.urandom(256) + yield "." + + # Msgpack + yield "
Msgpack:
" + import msgpack + binary = 'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv' + data = {"int": 1024*1024*1024, "float": 12345.67890, "text": "hello"*1024, "binary": binary} + with benchmark("pack 5K x 10 000", 0.78): + for i in range(10): + for y in range(1000): + data_packed = msgpack.packb(data) + yield "." + valid = """\x84\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohello\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa6binary\xda\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv""" + assert data_packed == valid, "%s
!=
%s" % (repr(data_packed), repr(valid)) + + with benchmark("unpack 5K x 10 000", 1.2): + for i in range(10): + for y in range(1000): + data_unpacked = msgpack.unpackb(data_packed) + yield "." + assert data == data_unpacked, "%s != %s" % (data_unpack, data) + + with benchmark("streaming unpack 5K x 10 000", 1.4): + for i in range(10): + unpacker = msgpack.Unpacker() + for y in range(1000): + unpacker.feed(data_packed) + for data_unpacked in unpacker: + pass + yield "." + assert data == data_unpacked, "%s != %s" % (data_unpack, data) + + # Db yield "
Db:
" from Db import Db diff --git a/src/Config.py b/src/Config.py index 7c99c42e..7b060a9b 100644 --- a/src/Config.py +++ b/src/Config.py @@ -8,7 +8,7 @@ class Config(object): def __init__(self, argv): self.version = "0.3.2" - self.rev = 396 + self.rev = 399 self.argv = argv self.action = None self.createParser() diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index a9d8f974..e15da4c7 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -13,6 +13,7 @@ from Debug import Debug from Connection import Connection from Config import config from Crypt import CryptConnection +from Crypt import CryptHash class ConnectionServer: @@ -36,10 +37,7 @@ class ConnectionServer: self.bytes_sent = 0 # Bittorrent style peerid - self.peer_id = "-ZN0%s-%s" % ( - config.version.replace(".", ""), - ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12)) - ) + self.peer_id = "-ZN0%s-%s" % (config.version.replace(".", ""), CryptHash.random(12, "base64")) # Check msgpack version if msgpack.version[0] == 0 and msgpack.version[1] < 4: diff --git a/src/Crypt/CryptHash.py b/src/Crypt/CryptHash.py index e71fa0e7..e25c06c6 100644 --- a/src/Crypt/CryptHash.py +++ b/src/Crypt/CryptHash.py @@ -1,4 +1,6 @@ import hashlib +import os +import base64 def sha1sum(file, blocksize=65536): @@ -19,6 +21,15 @@ def sha512sum(file, blocksize=65536): return hash.hexdigest()[0:64] # Truncate to 256bits is good enough +def random(length=64, encoding="hex"): + if encoding == "base64": # Characters: A-Za-z0-9 + hash = hashlib.sha512(os.urandom(256)).digest() + return base64.standard_b64encode(hash).replace("+", "").replace("/", "").replace("=", "")[0:length] + else: # Characters: a-f0-9 (faster) + return hashlib.sha512(os.urandom(256)).hexdigest()[0:length] + + + if __name__ == "__main__": import cStringIO as StringIO a = StringIO.StringIO() diff --git a/src/Site/Site.py b/src/Site/Site.py index 6152e0d8..ab6ba3e6 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -24,6 +24,7 @@ from Worker import WorkerManager from Debug import Debug from Content import ContentManager from SiteStorage import SiteStorage +from Crypt import CryptHash import SiteManager @@ -50,16 +51,12 @@ class Site: self.content_manager = ContentManager(self) # Load contents if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed) - self.settings["auth_key"] = ''.join( - random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24) - ) + self.settings["auth_key"] = CryptHash.random() self.log.debug("New auth key: %s" % self.settings["auth_key"]) self.saveSettings() if not self.settings.get("wrapper_key"): # To auth websocket permissions - self.settings["wrapper_key"] = ''.join( - random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12) - ) + self.settings["wrapper_key"] = CryptHash.random() self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"]) self.saveSettings() diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index e100e357..31c62d65 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -12,6 +12,7 @@ from Site import SiteManager from User import UserManager from Plugin import PluginManager from Ui.UiWebsocket import UiWebsocket +from Crypt import CryptHash status_texts = { 200: "200 OK", @@ -48,6 +49,10 @@ class UiRequest(object): path = re.sub("^http://zero[/]+", "/", path) # Remove begining http://zero/ for chrome extension path = re.sub("^http://", "/", path) # Remove begining http for chrome extension .bit access + if self.env["REQUEST_METHOD"] == "OPTIONS": + self.sendHeader() + return "" + if path == "/": return self.actionIndex() elif path.endswith("favicon.ico"): @@ -265,9 +270,7 @@ class UiRequest(object): # Create a new wrapper nonce that allows to get one html file without the wrapper def getWrapperNonce(self): - wrapper_nonce = ''.join( - random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24) - ) + wrapper_nonce = CryptHash.random() self.server.wrapper_nonces.append(wrapper_nonce) return wrapper_nonce diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index 797a6355..10ecad01 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -1,6 +1,7 @@ import logging import time import cgi +import socket from gevent.pywsgi import WSGIServer from gevent.pywsgi import WSGIHandler @@ -22,7 +23,6 @@ class UiWSGIHandler(WSGIHandler): self.kwargs = kwargs def run_application(self): - self.server.sockets[self.client_address] = self.socket if "HTTP_UPGRADE" in self.environ: # Websocket request try: ws_handler = WebSocketHandler(*self.args, **self.kwargs) @@ -32,20 +32,21 @@ class UiWSGIHandler(WSGIHandler): logging.error("UiWSGIHandler websocket error: %s" % Debug.formatException(err)) if config.debug: # Allow websocket errors to appear on /Debug import sys - del self.server.sockets[self.client_address] sys.modules["main"].DebugHook.handleError() else: # Standard HTTP request - # print self.application.__class__.__name__ try: super(UiWSGIHandler, self).run_application() except Exception, err: logging.error("UiWSGIHandler error: %s" % Debug.formatException(err)) if config.debug: # Allow websocket errors to appear on /Debug import sys - del self.server.sockets[self.client_address] sys.modules["main"].DebugHook.handleError() - if self.client_address in self.server.sockets: - del self.server.sockets[self.client_address] + + def handle(self): + # Save socket to be able to close them properly on exit + self.server.sockets[self.client_address] = self.socket + super(UiWSGIHandler, self).handle() + del self.server.sockets[self.client_address] class UiServer: @@ -131,12 +132,15 @@ class UiServer: sock_closed = 0 for sock in self.server.sockets.values(): try: - sock._sock.close() - sock.close() + sock.send("bye") + sock.shutdown(socket.SHUT_RDWR) + #sock._sock.close() + #sock.close() sock_closed += 1 - except Exception: - pass + except Exception, err: + self.log.debug("Http connection close error: %s" % err) self.log.debug("Socket closed: %s" % sock_closed) + time.sleep(0.1) self.server.socket.close() self.server.stop() diff --git a/src/Ui/media/Wrapper.css b/src/Ui/media/Wrapper.css index 67f4f99b..f22099e2 100644 --- a/src/Ui/media/Wrapper.css +++ b/src/Ui/media/Wrapper.css @@ -10,6 +10,7 @@ a { color: black } .button { padding: 5px 10px; margin-left: 10px; background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; border-radius: 2px; text-decoration: none; transition: all 0.5s; background-position: left center; } .button:hover { background-color: #FFF400; border-bottom: 2px solid #4D4D4C; transition: none } .button:active { position: relative; top: 1px } +.button:focus { outline: none } .button-Delete { background-color: #e74c3c; border-bottom-color: #c0392b; color: white } .button-Delete:hover { background-color: #FF5442; border-bottom-color: #8E2B21 } diff --git a/src/Ui/media/all.css b/src/Ui/media/all.css index a649c605..b29b9068 100644 --- a/src/Ui/media/all.css +++ b/src/Ui/media/all.css @@ -15,6 +15,7 @@ a { color: black } .button { padding: 5px 10px; margin-left: 10px; background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; -webkit-border-radius: 2px; -moz-border-radius: 2px; -o-border-radius: 2px; -ms-border-radius: 2px; border-radius: 2px ; text-decoration: none; -webkit-transition: all 0.5s; -moz-transition: all 0.5s; -o-transition: all 0.5s; -ms-transition: all 0.5s; transition: all 0.5s ; background-position: left center; } .button:hover { background-color: #FFF400; border-bottom: 2px solid #4D4D4C; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none } .button:active { position: relative; top: 1px } +.button:focus { outline: none } .button-Delete { background-color: #e74c3c; border-bottom-color: #c0392b; color: white } .button-Delete:hover { background-color: #FF5442; border-bottom-color: #8E2B21 }