oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Version 0.3.2, rev351, Sidebar to display site infos an modify settings, Per-site upload/download bytes statistics, Deny different origin media requests, Allow 10sec to finish query modifications, Websocket display errors to client instead of disconnecting, Allow specify notification id to server-side messages, Track every command response time

Browse source
This commit is contained in:
HelloZeroNet 2015-08-16 11:51:00 +02:00
parent b1c5b7d3a3
commit b83d6ba2ff
43 changed files with 8104 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/Config.py
View file

@ -7,8 +7,8 @@ import ConfigParser
class Config(object):
def __init__(self, argv):
self.version = "0.3.1"
self.rev = 338
self.version = "0.3.2"
self.rev = 351
self.argv = argv
self.action = None
self.createParser()

3
src/Connection/Connection.py
View file

@ -176,6 +176,9 @@ class Connection(object):
self.last_message_time = time.time()
if message.get("cmd") == "response": # New style response
if message["to"] in self.waiting_requests:
if self.last_send_time:
ping = time.time() - self.last_send_time
self.last_ping_delay = ping
self.waiting_requests[message["to"]].set(message) # Set the response to event
del self.waiting_requests[message["to"]]
elif message["to"] == 0: # Other peers handshake

4
src/Db/Db.py
View file

@ -12,14 +12,14 @@ opened_dbs = []
# Close idle databases to save some memory
def cleanup():
def dbCleanup():
while 1:
time.sleep(60 * 5)
for db in opened_dbs[:]:
if time.time() - db.last_query_time > 60 * 3:
db.close()
gevent.spawn(cleanup)
gevent.spawn(dbCleanup)
class Db:

21
src/File/FileRequest.py
View file

@ -139,10 +139,11 @@ class FileRequest(object):
with StreamingMsgpack.FilePart(file_path, "rb") as file:
file.seek(params["location"])
file.read_bytes = FILE_BUFF
file_size = os.fstat(file.fileno()).st_size
back = {
"body": file,
"size": os.fstat(file.fileno()).st_size,
"location": min(file.tell() + FILE_BUFF, os.fstat(file.fileno()).st_size)
"size": file_size,
"location": min(file.tell() + FILE_BUFF, file_size)
}
if config.debug_socket:
self.log.debug(
@ -150,8 +151,11 @@ class FileRequest(object):
(file_path, params["location"], back["location"])
)
self.response(back, streaming=True)
bytes_sent = min(FILE_BUFF, file_size - params["location"]) # Number of bytes we going to send
site.settings["bytes_sent"] = site.settings.get("bytes_sent", 0) + bytes_sent
if config.debug_socket:
self.log.debug("File %s sent" % file_path)
self.log.debug("File %s at position %s sent %s bytes" % (file_path, params["location"], bytes_sent))
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
@ -174,10 +178,11 @@ class FileRequest(object):
self.log.debug("Opening file: %s" % params["inner_path"])
with site.storage.open(params["inner_path"]) as file:
file.seek(params["location"])
stream_bytes = min(FILE_BUFF, os.fstat(file.fileno()).st_size-params["location"])
file_size = os.fstat(file.fileno()).st_size
stream_bytes = min(FILE_BUFF, file_size - params["location"])
back = {
"size": os.fstat(file.fileno()).st_size,
"location": min(file.tell() + FILE_BUFF, os.fstat(file.fileno()).st_size),
"size": file_size,
"location": min(file.tell() + FILE_BUFF, file_size),
"stream_bytes": stream_bytes
}
if config.debug_socket:
@ -187,8 +192,10 @@ class FileRequest(object):
)
self.response(back)
self.sendRawfile(file, read_bytes=FILE_BUFF)
site.settings["bytes_sent"] = site.settings.get("bytes_sent", 0) + stream_bytes
if config.debug_socket:
self.log.debug("File %s sent" % params["inner_path"])
self.log.debug("File %s at position %s sent %s bytes" % (params["inner_path"], params["location"], stream_bytes))
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port)

2
src/Peer/Peer.py
View file

@ -151,6 +151,7 @@ class Peer(object):
self.download_bytes += back["location"]
self.download_time += (time.time() - s)
self.site.settings["bytes_recv"] = self.site.settings.get("bytes_recv", 0) + back["location"]
buff.seek(0)
return buff
@ -177,6 +178,7 @@ class Peer(object):
self.download_bytes += back["location"]
self.download_time += (time.time() - s)
self.site.settings["bytes_recv"] = self.site.settings.get("bytes_recv", 0) + back["location"]
buff.seek(0)
return buff

13
src/Site/Site.py
View file

@ -137,11 +137,11 @@ class Site:
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
gevent.joinall(include_threads)
self.log.debug("%s: Includes downloaded" % inner_path)
self.log.debug("%s: Includes download ended" % inner_path)
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
self.log.debug("%s: All file downloaded in %.2fs" % (inner_path, time.time() - s))
self.log.debug("%s: DownloadContent ended in %.2fs" % (inner_path, time.time() - s))
return True
@ -159,7 +159,10 @@ class Site:
# Download all files of the site
@util.Noparallel(blocking=False)
def download(self, check_size=False, blind_includes=False):
self.log.debug("Start downloading, bad_files: %s, check_size: %s, blind_includes: %s" % (self.bad_files, check_size, blind_includes))
self.log.debug(
"Start downloading, bad_files: %s, check_size: %s, blind_includes: %s" %
(self.bad_files, check_size, blind_includes)
)
gevent.spawn(self.announce)
if check_size: # Check the size first
valid = self.downloadContent(download_files=False) # Just download content.json files
@ -221,7 +224,7 @@ class Site:
for i in range(3):
updaters.append(gevent.spawn(self.updater, peers_try, queried, since))
gevent.joinall(updaters, timeout=5) # Wait 5 sec to workers
gevent.joinall(updaters, timeout=10) # Wait 10 sec to workers done query modifications
time.sleep(0.1)
self.log.debug("Queried listModifications from: %s" % queried)
return queried
@ -420,7 +423,7 @@ class Site:
elif self.settings["serving"] is False: # Site not serving
return False
else: # Wait until file downloaded
self.bad_files[inner_path] = self.bad_files.get(inner_path,0)+1 # Mark as bad file
self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file
if not self.content_manager.contents.get("content.json"): # No content.json, download it first!
self.log.debug("Need content.json first")
gevent.spawn(self.announce)

5
src/Site/SiteStorage.py
View file

@ -210,6 +210,11 @@ class SiteStorage:
raise Exception("File not allowed: %s" % file_path)
return file_path
# Get site dir relative path
def getInnerPath(self, path):
inner_path = re.sub("^%s/" % re.escape(self.directory), "", path)
return inner_path
# Verify all files sha512sum using content.json
def verifyFiles(self, quick_check=False): # Fast = using file size
bad_files = []

24
src/Ui/UiWebsocket.py
View file

@ -57,14 +57,18 @@ class UiWebsocket(object):
while True:
try:
message = ws.receive()
if message:
self.handleRequest(message)
except Exception, err:
if err.message != 'Connection is already closed':
self.log.error("WebSocket receive error: %s" % err)
return "Bye." # Close connection
if message:
try:
self.handleRequest(message)
except Exception, err:
if config.debug: # Allow websocket errors to appear on /Debug
sys.modules["main"].DebugHook.handleError()
self.log.error("WebSocket error: %s" % Debug.formatException(err))
return "Bye."
self.log.error("WebSocket handleRequest error: %s" % err)
self.cmd("error", "Internal error: %s" % err)
# Event in a channel
def event(self, channel, *params):
@ -138,8 +142,10 @@ class UiWebsocket(object):
func(req["id"], **params)
elif type(params) is list:
func(req["id"], *params)
else:
elif params:
func(req["id"], params)
else:
func(req["id"])
# Format site info
def formatSiteInfo(self, site, create_user=True):
@ -170,7 +176,7 @@ class UiWebsocket(object):
"bad_files": len(site.bad_files),
"size_limit": site.getSizeLimit(),
"next_size_limit": site.getNextSizeLimit(),
"peers": site.settings.get("peers", len(site.peers)),
"peers": max(site.settings.get("peers", 0), len(site.peers)),
"started_task_num": site.worker_manager.started_task_num,
"tasks": len(site.worker_manager.tasks),
"workers": len(site.worker_manager.workers),
@ -404,7 +410,7 @@ class UiWebsocket(object):
if auth_address == cert["auth_address"]:
active = domain
title = cert["auth_user_name"] + "@" + domain
if domain in accepted_domains:
if domain in accepted_domains or not accepted_domains:
accounts.append([domain, title, ""])
else:
accounts.append([domain, title, "disabled"])
@ -527,7 +533,7 @@ class UiWebsocket(object):
gevent.spawn(new_site.announce)
def actionSiteSetLimit(self, to, size_limit):
self.site.settings["size_limit"] = size_limit
self.site.settings["size_limit"] = int(size_limit)
self.site.saveSettings()
self.response(to, "Site size limit changed to %sMB" % size_limit)
self.site.download(blind_includes=True)

8
src/Ui/media/Wrapper.coffee
View file

@ -49,7 +49,11 @@ class Wrapper
else
@sendInner message # Pass message to inner frame
else if cmd == "notification" # Display notification
@notifications.add("notification-#{message.id}", message.params[0], message.params[1], message.params[2])
type = message.params[0]
id = "notification-#{message.id}"
if "-" in message.params[0] # - in first param: message id definied
[id, type] = message.params[0].split("-")
@notifications.add(id, type, message.params[1], message.params[2])
else if cmd == "prompt" # Prompt input
@displayPrompt message.params[0], message.params[1], message.params[2], (res) =>
@ws.response message.id, res
@ -57,6 +61,8 @@ class Wrapper
@sendInner message # Pass to inner frame
if message.params.address == @address # Current page
@setSiteInfo message.params
else if cmd == "error"
@notifications.add("notification-#{message.id}", "error", message.params, 0)
else if cmd == "updating" # Close connection
@ws.ws.close()
@ws.onCloseWebsocket(null, 4000)

6
src/Ui/media/Wrapper.css
View file

@ -7,13 +7,17 @@ a { color: black }
#inner-iframe { width: 100%; height: 100%; position: absolute; border: 0px } /*; transition: all 0.8s cubic-bezier(0.68, -0.55, 0.265, 1.55), opacity 0.8s ease-in-out*/
#inner-iframe.back { transform: scale(0.95) translate(-300px, 0px); opacity: 0.4 }
.button { padding: 5px 10px; margin-left: 10px; background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; border-radius: 2px; text-decoration: none; transition: all 0.5s; }
.button { padding: 5px 10px; margin-left: 10px; background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; border-radius: 2px; text-decoration: none; transition: all 0.5s; background-position: left center; }
.button:hover { background-color: #FFF400; border-bottom: 2px solid #4D4D4C; transition: none }
.button:active { position: relative; top: 1px }
.button-Delete { background-color: #e74c3c; border-bottom-color: #c0392b; color: white }
.button-Delete:hover { background-color: #FF5442; border-bottom-color: #8E2B21 }
.button.loading {
color: rgba(0,0,0,0); background: #999 url(img/loading.gif) no-repeat center center;
transition: all 0.5s ease-out ; pointer-events: none; border-bottom: 2px solid #666
}
/* Fixbutton */

6
src/Ui/media/all.css
View file

@ -12,13 +12,17 @@ a { color: black }
#inner-iframe { width: 100%; height: 100%; position: absolute; border: 0px } /*; transition: all 0.8s cubic-bezier(0.68, -0.55, 0.265, 1.55), opacity 0.8s ease-in-out*/
#inner-iframe.back { -webkit-transform: scale(0.95) translate(-300px, 0px); -moz-transform: scale(0.95) translate(-300px, 0px); -o-transform: scale(0.95) translate(-300px, 0px); -ms-transform: scale(0.95) translate(-300px, 0px); transform: scale(0.95) translate(-300px, 0px) ; opacity: 0.4 }
.button { padding: 5px 10px; margin-left: 10px; background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; -webkit-border-radius: 2px; -moz-border-radius: 2px; -o-border-radius: 2px; -ms-border-radius: 2px; border-radius: 2px ; text-decoration: none; -webkit-transition: all 0.5s; -moz-transition: all 0.5s; -o-transition: all 0.5s; -ms-transition: all 0.5s; transition: all 0.5s ; }
.button { padding: 5px 10px; margin-left: 10px; background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; -webkit-border-radius: 2px; -moz-border-radius: 2px; -o-border-radius: 2px; -ms-border-radius: 2px; border-radius: 2px ; text-decoration: none; -webkit-transition: all 0.5s; -moz-transition: all 0.5s; -o-transition: all 0.5s; -ms-transition: all 0.5s; transition: all 0.5s ; background-position: left center; }
.button:hover { background-color: #FFF400; border-bottom: 2px solid #4D4D4C; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none }
.button:active { position: relative; top: 1px }
.button-Delete { background-color: #e74c3c; border-bottom-color: #c0392b; color: white }
.button-Delete:hover { background-color: #FF5442; border-bottom-color: #8E2B21 }
.button.loading {
color: rgba(0,0,0,0); background: #999 url(img/loading.gif) no-repeat center center;
-webkit-transition: all 0.5s ease-out ; -moz-transition: all 0.5s ease-out ; -o-transition: all 0.5s ease-out ; -ms-transition: all 0.5s ease-out ; transition: all 0.5s ease-out ; pointer-events: none; border-bottom: 2px solid #666
}
/* Fixbutton */

12
src/Ui/media/all.js
View file

@ -758,6 +758,7 @@ jQuery.extend( jQuery.easing,
(function() {
var Wrapper, origin, proto, ws_url,
__bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; },
__indexOf = [].indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; },
__slice = [].slice;
Wrapper = (function() {
@ -810,7 +811,7 @@ jQuery.extend( jQuery.easing,
}
Wrapper.prototype.onMessageWebsocket = function(e) {
var cmd, message;
var cmd, id, message, type, _ref;
message = JSON.parse(e.data);
cmd = message.cmd;
if (cmd === "response") {
@ -820,7 +821,12 @@ jQuery.extend( jQuery.easing,
return this.sendInner(message);
}
} else if (cmd === "notification") {
return this.notifications.add("notification-" + message.id, message.params[0], message.params[1], message.params[2]);
type = message.params[0];
id = "notification-" + message.id;
if (__indexOf.call(message.params[0], "-") >= 0) {
_ref = message.params[0].split("-"), id = _ref[0], type = _ref[1];
}
return this.notifications.add(id, type, message.params[1], message.params[2]);
} else if (cmd === "prompt") {
return this.displayPrompt(message.params[0], message.params[1], message.params[2], (function(_this) {
return function(res) {
@ -832,6 +838,8 @@ jQuery.extend( jQuery.easing,
if (message.params.address === this.address) {
return this.setSiteInfo(message.params);
}
} else if (cmd === "error") {
return this.notifications.add("notification-" + message.id, "error", message.params, 0);
} else if (cmd === "updating") {
this.ws.ws.close();
return this.ws.onCloseWebsocket(null, 4000);

BIN
src/Ui/media/img/loading-circle.gif Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.3 KiB

BIN
src/Ui/media/img/loading.gif Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 723 B

4
src/util/RateLimit.py
View file

@ -78,14 +78,14 @@ def call(event, allowed_again=10, func=None, *args, **kwargs):
# Cleanup expired events every 3 minutes
def cleanup():
def rateLimitCleanup():
while 1:
expired = time.time() - 60 * 2 # Cleanup if older than 2 minutes
for event in called_db.keys():
if called_db[event] < expired:
del called_db[event]
time.sleep(60 * 3) # Every 3 minutes
gevent.spawn(cleanup)
gevent.spawn(rateLimitCleanup)
if __name__ == "__main__":