oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rev338, Possible to use pure-python msgpack unpacker to save memory, Streaming file download hangup fix, Clone databases after 3 minute idle, Many site size limit related bugfixes, UiMedia served from same domain to allow ajax access, Don't allow to load resources from other domain, Site size increase ask dialog displayed again, Changed fixbutton to more Consolas-like Monaco font, Boost json files priority on download

Browse source
This commit is contained in:
HelloZeroNet 2015-08-06 00:51:25 +02:00
parent cd04abe521
commit 84e3f00aac
15 changed files with 82 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Config.py
View file

@ -8,7 +8,7 @@ class Config(object):
def __init__(self, argv): def __init__(self, argv):
self.version = "0.3.1" self.version = "0.3.1"
self.rev = 330 self.rev = 338
self.argv = argv self.argv = argv
self.action = None self.action = None
self.createParser() self.createParser()
@ -130,6 +130,8 @@ class Config(object):
type='bool', choices=[True, False], default=False) type='bool', choices=[True, False], default=False)
self.parser.add_argument('--stream_downloads', help='Stream download directly to files (experimental)', self.parser.add_argument('--stream_downloads', help='Stream download directly to files (experimental)',
type='bool', choices=[True, False], default=False) type='bool', choices=[True, False], default=False)
self.parser.add_argument("--msgpack_purepython", help='Use less memory, but a bit more CPU power',
type='bool', choices=[True, False], default=False)
self.parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript, self.parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript,
metavar='executable_path') metavar='executable_path')

2
src/Connection/Connection.py
View file

@ -237,6 +237,8 @@ class Connection(object):
if read_bytes <= 0: if read_bytes <= 0:
break break
buff = self.sock.recv(16 * 1024) buff = self.sock.recv(16 * 1024)
if not buff:
break
buff_len = len(buff) buff_len = len(buff)
read_bytes -= buff_len read_bytes -= buff_len
file.write(buff) file.write(buff)

26
src/Db/Db.py
View file

@ -4,9 +4,23 @@ import time
import logging import logging
import re import re
import os import os
import gevent
from DbCursor import DbCursor from DbCursor import DbCursor
opened_dbs = []
# Close idle databases to save some memory
def cleanup():
while 1:
time.sleep(60 * 5)
for db in opened_dbs[:]:
if time.time() - db.last_query_time > 60 * 3:
db.close()
gevent.spawn(cleanup)
class Db: class Db:
@ -22,8 +36,15 @@ class Db:
self.collect_stats = False self.collect_stats = False
self.query_stats = {} self.query_stats = {}
self.db_keyvalues = {} self.db_keyvalues = {}
self.last_query_time = time.time()
def __repr__(self):
return "<Db:%s>" % self.db_path
def connect(self): def connect(self):
if self not in opened_dbs:
opened_dbs.append(self)
self.log.debug("Connecting to %s (sqlite version: %s)..." % (self.db_path, sqlite3.version)) self.log.debug("Connecting to %s (sqlite version: %s)..." % (self.db_path, sqlite3.version))
if not os.path.isdir(self.db_dir): # Directory not exist yet if not os.path.isdir(self.db_dir): # Directory not exist yet
os.makedirs(self.db_dir) os.makedirs(self.db_dir)
@ -41,16 +62,21 @@ class Db:
# Execute query using dbcursor # Execute query using dbcursor
def execute(self, query, params=None): def execute(self, query, params=None):
self.last_query_time = time.time()
if not self.conn: if not self.conn:
self.connect() self.connect()
return self.cur.execute(query, params) return self.cur.execute(query, params)
def close(self): def close(self):
if self in opened_dbs:
opened_dbs.remove(self)
self.log.debug("Closing") self.log.debug("Closing")
if self.cur: if self.cur:
self.cur.close() self.cur.close()
if self.conn: if self.conn:
self.conn.close() self.conn.close()
self.conn = None
self.cur = None
# Gets a cursor object to database # Gets a cursor object to database
# Return: Cursor class # Return: Cursor class

4
src/Peer/Peer.py
View file

@ -73,7 +73,7 @@ class Peer(object):
return self.connection return self.connection
def __str__(self): def __str__(self):
return "Peer %-12s" % self.ip return "Peer:%-12s" % self.ip
def __repr__(self): def __repr__(self):
return "<%s>" % self.__str__() return "<%s>" % self.__str__()
@ -126,7 +126,7 @@ class Peer(object):
# Get a file content from peer # Get a file content from peer
def getFile(self, site, inner_path): def getFile(self, site, inner_path):
# Use streamFile if client supports it # Use streamFile if client supports it
if config.stream_downloads and self.connection and self.connection.handshake["rev"] > 310: if config.stream_downloads and self.connection and self.connection.handshake and self.connection.handshake["rev"] > 310:
return self.streamFile(site, inner_path) return self.streamFile(site, inner_path)
location = 0 location = 0

20
src/Site/Site.py
View file

@ -158,8 +158,8 @@ class Site:
# Download all files of the site # Download all files of the site
@util.Noparallel(blocking=False) @util.Noparallel(blocking=False)
def download(self, check_size=False): def download(self, check_size=False, blind_includes=False):
self.log.debug("Start downloading...%s" % self.bad_files) self.log.debug("Start downloading, bad_files: %s, check_size: %s, blind_includes: %s" % (self.bad_files, check_size, blind_includes))
gevent.spawn(self.announce) gevent.spawn(self.announce)
if check_size: # Check the size first if check_size: # Check the size first
valid = self.downloadContent(download_files=False) # Just download content.json files valid = self.downloadContent(download_files=False) # Just download content.json files
@ -167,10 +167,14 @@ class Site:
return False # Cant download content.jsons or size is not fits return False # Cant download content.jsons or size is not fits
# Download everything # Download everything
found = self.downloadContent("content.json") valid = self.downloadContent("content.json")
self.checkModifications(0) # Download multiuser blind includes
return found if valid and blind_includes:
self.checkModifications(0) # Download multiuser blind includes
self.retryBadFiles()
return valid
# Update worker, try to find client that supports listModifications command # Update worker, try to find client that supports listModifications command
def updater(self, peers_try, queried, since): def updater(self, peers_try, queried, since):
@ -189,7 +193,9 @@ class Site:
queried.append(peer) queried.append(peer)
for inner_path, modified in res["modified_files"].iteritems(): # Check if the peer has newer files than we for inner_path, modified in res["modified_files"].iteritems(): # Check if the peer has newer files than we
content = self.content_manager.contents.get(inner_path) content = self.content_manager.contents.get(inner_path)
if not content or modified > content["modified"]: # We dont have this file or we have older if (not content or modified > content["modified"]) and inner_path not in self.bad_files:
self.log.debug("New modified file from %s: %s" % (peer, inner_path))
# We dont have this file or we have older
self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file
gevent.spawn(self.downloadContent, inner_path) # Download the content.json + the changed files gevent.spawn(self.downloadContent, inner_path) # Download the content.json + the changed files
@ -414,7 +420,7 @@ class Site:
elif self.settings["serving"] is False: # Site not serving elif self.settings["serving"] is False: # Site not serving
return False return False
else: # Wait until file downloaded else: # Wait until file downloaded
self.bad_files[inner_path] = True # Mark as bad file self.bad_files[inner_path] = self.bad_files.get(inner_path,0)+1 # Mark as bad file
if not self.content_manager.contents.get("content.json"): # No content.json, download it first! if not self.content_manager.contents.get("content.json"): # No content.json, download it first!
self.log.debug("Need content.json first") self.log.debug("Need content.json first")
gevent.spawn(self.announce) gevent.spawn(self.announce)

7
src/Site/SiteManager.py
View file

@ -78,9 +78,12 @@ class SiteManager(object):
if not site.settings["serving"]: # Maybe it was deleted before if not site.settings["serving"]: # Maybe it was deleted before
site.settings["serving"] = True site.settings["serving"] = True
site.saveSettings() site.saveSettings()
if all_file: # Also download user files on first sync
site.download(blind_includes=True)
else:
if all_file:
site.download()
if all_file:
site.download()
return site return site
def delete(self, address): def delete(self, address):

6
src/Ui/UiRequest.py
View file

@ -53,6 +53,10 @@ class UiRequest(object):
# Media # Media
elif path.startswith("/uimedia/"): elif path.startswith("/uimedia/"):
return self.actionUiMedia(path) return self.actionUiMedia(path)
elif "/uimedia/" in path:
# uimedia within site dir (for chrome extension)
path = re.sub(".*?/uimedia/", "/uimedia/", path)
return self.actionUiMedia(path)
elif path.startswith("/media"): elif path.startswith("/media"):
return self.actionSiteMedia(path) return self.actionSiteMedia(path)
# Websocket # Websocket
@ -258,6 +262,8 @@ class UiRequest(object):
# Returns if media request allowed from that referer # Returns if media request allowed from that referer
def isMediaRequestAllowed(self, site_address, referer): def isMediaRequestAllowed(self, site_address, referer):
if not re.sub("^http[s]{0,1}://", "", referer).startswith(self.env["HTTP_HOST"]):
return False
referer_path = re.sub("http[s]{0,1}://.*?/", "/", referer).replace("/media", "") # Remove site address referer_path = re.sub("http[s]{0,1}://.*?/", "/", referer).replace("/media", "") # Remove site address
return referer_path.startswith("/" + site_address) return referer_path.startswith("/" + site_address)

2
src/Ui/UiWebsocket.py
View file

@ -530,7 +530,7 @@ class UiWebsocket(object):
self.site.settings["size_limit"] = size_limit self.site.settings["size_limit"] = size_limit
self.site.saveSettings() self.site.saveSettings()
self.response(to, "Site size limit changed to %sMB" % size_limit) self.response(to, "Site size limit changed to %sMB" % size_limit)
self.site.download() self.site.download(blind_includes=True)
def actionServerUpdate(self, to): def actionServerUpdate(self, to):
self.cmd("updating") self.cmd("updating")

8
src/Ui/media/Wrapper.coffee
View file

@ -290,9 +290,9 @@ class Wrapper
@site_error = "No peers found" @site_error = "No peers found"
@loading.printLine "No peers found" @loading.printLine "No peers found"
if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").indexOf("?") == -1 # First site info and mainpage if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").replace("?wrapper=False", "").indexOf("?") == -1 # First site info and mainpage
if site_info.size_limit*1.1 < site_info.next_size_limit # Need upgrade soon if site_info.size_limit*1.1 < site_info.next_size_limit # Need upgrade soon
@actionConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", => @displayConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", =>
@ws.cmd "siteSetLimit", [site_info.next_size_limit], (res) => @ws.cmd "siteSetLimit", [site_info.next_size_limit], (res) =>
@notifications.add("size_limit", "done", res, 5000) @notifications.add("size_limit", "done", res, 5000)
return false return false
@ -323,7 +323,9 @@ class Wrapper
@loading.printLine res @loading.printLine res
@inner_loaded = false # Inner frame not loaded, just a 404 page displayed @inner_loaded = false # Inner frame not loaded, just a 404 page displayed
if reload if reload
$("iframe").attr "src", $("iframe").attr("src")+"&"+(+new Date) # Reload iframe src = $("iframe").attr("src")
$("iframe").attr "src", ""
$("iframe").attr "src", src
return false return false

4
src/Ui/media/Wrapper.css
View file

@ -19,7 +19,7 @@ a { color: black }
.fixbutton { .fixbutton {
position: absolute; right: 35px; top: 15px; width: 40px; z-index: 999; position: absolute; right: 35px; top: 15px; width: 40px; z-index: 999;
text-align: center; color: white; font-family: Consolas, Menlo, monospace; font-size: 25px; text-align: center; color: white; font-family: Consolas, Monaco, monospace; font-size: 25px;
} }
.fixbutton-bg { .fixbutton-bg {
border-radius: 80px; background-color: rgba(180, 180, 180, 0.5); cursor: pointer; border-radius: 80px; background-color: rgba(180, 180, 180, 0.5); cursor: pointer;
@ -27,7 +27,7 @@ a { color: black }
/*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); */ /*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); */
} }
.fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; perspective: 1000px; line-height: 0px; padding-top: 20px } .fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; perspective: 1000px; line-height: 0px; padding-top: 20px }
.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 48px; line-height: 32px } .fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px }
.fixbutton-bg:hover { background-color: #AF3BFF } .fixbutton-bg:hover { background-color: #AF3BFF }
.fixbutton-bg:active { background-color: #9E2FEA; top: 1px; transition: none } .fixbutton-bg:active { background-color: #9E2FEA; top: 1px; transition: none }

4
src/Ui/media/all.css
View file

@ -24,7 +24,7 @@ a { color: black }
.fixbutton { .fixbutton {
position: absolute; right: 35px; top: 15px; width: 40px; z-index: 999; position: absolute; right: 35px; top: 15px; width: 40px; z-index: 999;
text-align: center; color: white; font-family: Consolas, Menlo, monospace; font-size: 25px; text-align: center; color: white; font-family: Consolas, Monaco, monospace; font-size: 25px;
} }
.fixbutton-bg { .fixbutton-bg {
-webkit-border-radius: 80px; -moz-border-radius: 80px; -o-border-radius: 80px; -ms-border-radius: 80px; border-radius: 80px ; background-color: rgba(180, 180, 180, 0.5); cursor: pointer; -webkit-border-radius: 80px; -moz-border-radius: 80px; -o-border-radius: 80px; -ms-border-radius: 80px; border-radius: 80px ; background-color: rgba(180, 180, 180, 0.5); cursor: pointer;
@ -32,7 +32,7 @@ a { color: black }
/*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* -webkit-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -moz-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -o-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -ms-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1) ; */ /*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* -webkit-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -moz-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -o-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -ms-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1) ; */
} }
.fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px } .fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px }
.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 48px; line-height: 32px } .fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px }
.fixbutton-bg:hover { background-color: #AF3BFF } .fixbutton-bg:hover { background-color: #AF3BFF }
.fixbutton-bg:active { background-color: #9E2FEA; top: 1px; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none } .fixbutton-bg:active { background-color: #9E2FEA; top: 1px; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none }

11
src/Ui/media/all.js
View file

@ -1140,9 +1140,9 @@ jQuery.extend( jQuery.easing,
this.loading.printLine("No peers found"); this.loading.printLine("No peers found");
} }
} }
if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").indexOf("?") === -1) { if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").replace("?wrapper=False", "").indexOf("?") === -1) {
if (site_info.size_limit * 1.1 < site_info.next_size_limit) { if (site_info.size_limit * 1.1 < site_info.next_size_limit) {
this.actionConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) { this.displayConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) {
return function() { return function() {
_this.ws.cmd("siteSetLimit", [site_info.next_size_limit], function(res) { _this.ws.cmd("siteSetLimit", [site_info.next_size_limit], function(res) {
return _this.notifications.add("size_limit", "done", res, 5000); return _this.notifications.add("size_limit", "done", res, 5000);
@ -1184,10 +1184,13 @@ jQuery.extend( jQuery.easing,
} }
this.ws.cmd("siteSetLimit", [size_limit], (function(_this) { this.ws.cmd("siteSetLimit", [size_limit], (function(_this) {
return function(res) { return function(res) {
var src;
_this.loading.printLine(res); _this.loading.printLine(res);
_this.inner_loaded = false; _this.inner_loaded = false;
if (reload) { if (reload) {
return $("iframe").attr("src", $("iframe").attr("src") + "&" + (+(new Date))); src = $("iframe").attr("src");
$("iframe").attr("src", "");
return $("iframe").attr("src", src);
} }
}; };
})(this)); })(this));
@ -1234,4 +1237,4 @@ jQuery.extend( jQuery.easing,
window.wrapper = new Wrapper(ws_url); window.wrapper = new Wrapper(ws_url);
}).call(this); }).call(this);

4
src/Ui/template/wrapper.html
View file

@ -6,7 +6,7 @@
<title>{title} - ZeroNet</title> <title>{title} - ZeroNet</title>
<meta charset="utf-8"> <meta charset="utf-8">
<meta http-equiv="content-type" content="text/html; charset=utf-8" /> <meta http-equiv="content-type" content="text/html; charset=utf-8" />
<link rel="stylesheet" href="{server_url}/uimedia/all.css?rev={rev}" /> <link rel="stylesheet" href="/uimedia/all.css?rev={rev}" />
{meta_tags} {meta_tags}
</head> </head>
<body style="{body_style}"> <body style="{body_style}">
@ -57,7 +57,7 @@ permissions = {permissions}
show_loadingscreen = {show_loadingscreen} show_loadingscreen = {show_loadingscreen}
server_url = '{server_url}' server_url = '{server_url}'
</script> </script>
<script type="text/javascript" src="{server_url}/uimedia/all.js?rev={rev}"></script> <script type="text/javascript" src="/uimedia/all.js?rev={rev}"></script>
</body> </body>
</html> </html>

4
src/Worker/WorkerManager.py
View file

@ -78,7 +78,9 @@ class WorkerManager:
return 9998 # index.html also important return 9998 # index.html also important
priority = task["priority"] priority = task["priority"]
if task["inner_path"].endswith(".js") or task["inner_path"].endswith(".css"): if task["inner_path"].endswith(".js") or task["inner_path"].endswith(".css"):
priority += 1 # download js and css files first priority += 2 # boost js and css files priority
elif task["inner_path"].endswith(".json"):
priority += 1 # boost json files priority
return priority - task["workers_num"] # Prefer more priority and less workers return priority - task["workers_num"] # Prefer more priority and less workers
# Returns the next free or less worked task # Returns the next free or less worked task

3
src/main.py
View file

@ -65,6 +65,9 @@ config.parse() # Parse again to add plugin configuration options
# Log current config # Log current config
logging.debug("Config: %s" % config) logging.debug("Config: %s" % config)
# Use pure-python implementation of msgpack to save CPU
if config.msgpack_purepython:
os.environ["MSGPACK_PUREPYTHON"] = "True"
# Socks Proxy monkey patch # Socks Proxy monkey patch
if config.proxy: if config.proxy: