From 84e3f00aac97b46577e188ff977411ebde6b2518 Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Thu, 6 Aug 2015 00:51:25 +0200 Subject: [PATCH] rev338, Possible to use pure-python msgpack unpacker to save memory, Streaming file download hangup fix, Clone databases after 3 minute idle, Many site size limit related bugfixes, UiMedia served from same domain to allow ajax access, Don't allow to load resources from other domain, Site size increase ask dialog displayed again, Changed fixbutton to more Consolas-like Monaco font, Boost json files priority on download --- src/Config.py | 4 +++- src/Connection/Connection.py | 2 ++ src/Db/Db.py | 26 ++++++++++++++++++++++++++ src/Peer/Peer.py | 4 ++-- src/Site/Site.py | 20 +++++++++++++------- src/Site/SiteManager.py | 7 +++++-- src/Ui/UiRequest.py | 6 ++++++ src/Ui/UiWebsocket.py | 2 +- src/Ui/media/Wrapper.coffee | 8 +++++--- src/Ui/media/Wrapper.css | 4 ++-- src/Ui/media/all.css | 4 ++-- src/Ui/media/all.js | 11 +++++++---- src/Ui/template/wrapper.html | 4 ++-- src/Worker/WorkerManager.py | 4 +++- src/main.py | 3 +++ 15 files changed, 82 insertions(+), 27 deletions(-) diff --git a/src/Config.py b/src/Config.py index 24dcbde7..47ac9919 100644 --- a/src/Config.py +++ b/src/Config.py @@ -8,7 +8,7 @@ class Config(object): def __init__(self, argv): self.version = "0.3.1" - self.rev = 330 + self.rev = 338 self.argv = argv self.action = None self.createParser() @@ -130,6 +130,8 @@ class Config(object): type='bool', choices=[True, False], default=False) self.parser.add_argument('--stream_downloads', help='Stream download directly to files (experimental)', type='bool', choices=[True, False], default=False) + self.parser.add_argument("--msgpack_purepython", help='Use less memory, but a bit more CPU power', + type='bool', choices=[True, False], default=False) self.parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript, metavar='executable_path') diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 8c166d4c..303faed0 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -237,6 +237,8 @@ class Connection(object): if read_bytes <= 0: break buff = self.sock.recv(16 * 1024) + if not buff: + break buff_len = len(buff) read_bytes -= buff_len file.write(buff) diff --git a/src/Db/Db.py b/src/Db/Db.py index 4f2034b9..34713c6e 100644 --- a/src/Db/Db.py +++ b/src/Db/Db.py @@ -4,9 +4,23 @@ import time import logging import re import os +import gevent from DbCursor import DbCursor +opened_dbs = [] + + +# Close idle databases to save some memory +def cleanup(): + while 1: + time.sleep(60 * 5) + for db in opened_dbs[:]: + if time.time() - db.last_query_time > 60 * 3: + db.close() + +gevent.spawn(cleanup) + class Db: @@ -22,8 +36,15 @@ class Db: self.collect_stats = False self.query_stats = {} self.db_keyvalues = {} + self.last_query_time = time.time() + + def __repr__(self): + return "" % self.db_path def connect(self): + if self not in opened_dbs: + opened_dbs.append(self) + self.log.debug("Connecting to %s (sqlite version: %s)..." % (self.db_path, sqlite3.version)) if not os.path.isdir(self.db_dir): # Directory not exist yet os.makedirs(self.db_dir) @@ -41,16 +62,21 @@ class Db: # Execute query using dbcursor def execute(self, query, params=None): + self.last_query_time = time.time() if not self.conn: self.connect() return self.cur.execute(query, params) def close(self): + if self in opened_dbs: + opened_dbs.remove(self) self.log.debug("Closing") if self.cur: self.cur.close() if self.conn: self.conn.close() + self.conn = None + self.cur = None # Gets a cursor object to database # Return: Cursor class diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 6e70114c..ba7089c4 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -73,7 +73,7 @@ class Peer(object): return self.connection def __str__(self): - return "Peer %-12s" % self.ip + return "Peer:%-12s" % self.ip def __repr__(self): return "<%s>" % self.__str__() @@ -126,7 +126,7 @@ class Peer(object): # Get a file content from peer def getFile(self, site, inner_path): # Use streamFile if client supports it - if config.stream_downloads and self.connection and self.connection.handshake["rev"] > 310: + if config.stream_downloads and self.connection and self.connection.handshake and self.connection.handshake["rev"] > 310: return self.streamFile(site, inner_path) location = 0 diff --git a/src/Site/Site.py b/src/Site/Site.py index 85cd07b8..7bc56e6a 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -158,8 +158,8 @@ class Site: # Download all files of the site @util.Noparallel(blocking=False) - def download(self, check_size=False): - self.log.debug("Start downloading...%s" % self.bad_files) + def download(self, check_size=False, blind_includes=False): + self.log.debug("Start downloading, bad_files: %s, check_size: %s, blind_includes: %s" % (self.bad_files, check_size, blind_includes)) gevent.spawn(self.announce) if check_size: # Check the size first valid = self.downloadContent(download_files=False) # Just download content.json files @@ -167,10 +167,14 @@ class Site: return False # Cant download content.jsons or size is not fits # Download everything - found = self.downloadContent("content.json") - self.checkModifications(0) # Download multiuser blind includes + valid = self.downloadContent("content.json") - return found + if valid and blind_includes: + self.checkModifications(0) # Download multiuser blind includes + + self.retryBadFiles() + + return valid # Update worker, try to find client that supports listModifications command def updater(self, peers_try, queried, since): @@ -189,7 +193,9 @@ class Site: queried.append(peer) for inner_path, modified in res["modified_files"].iteritems(): # Check if the peer has newer files than we content = self.content_manager.contents.get(inner_path) - if not content or modified > content["modified"]: # We dont have this file or we have older + if (not content or modified > content["modified"]) and inner_path not in self.bad_files: + self.log.debug("New modified file from %s: %s" % (peer, inner_path)) + # We dont have this file or we have older self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file gevent.spawn(self.downloadContent, inner_path) # Download the content.json + the changed files @@ -414,7 +420,7 @@ class Site: elif self.settings["serving"] is False: # Site not serving return False else: # Wait until file downloaded - self.bad_files[inner_path] = True # Mark as bad file + self.bad_files[inner_path] = self.bad_files.get(inner_path,0)+1 # Mark as bad file if not self.content_manager.contents.get("content.json"): # No content.json, download it first! self.log.debug("Need content.json first") gevent.spawn(self.announce) diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 34ba0562..1b61a5e3 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -78,9 +78,12 @@ class SiteManager(object): if not site.settings["serving"]: # Maybe it was deleted before site.settings["serving"] = True site.saveSettings() + if all_file: # Also download user files on first sync + site.download(blind_includes=True) + else: + if all_file: + site.download() - if all_file: - site.download() return site def delete(self, address): diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 7e37fcef..bf29472b 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -53,6 +53,10 @@ class UiRequest(object): # Media elif path.startswith("/uimedia/"): return self.actionUiMedia(path) + elif "/uimedia/" in path: + # uimedia within site dir (for chrome extension) + path = re.sub(".*?/uimedia/", "/uimedia/", path) + return self.actionUiMedia(path) elif path.startswith("/media"): return self.actionSiteMedia(path) # Websocket @@ -258,6 +262,8 @@ class UiRequest(object): # Returns if media request allowed from that referer def isMediaRequestAllowed(self, site_address, referer): + if not re.sub("^http[s]{0,1}://", "", referer).startswith(self.env["HTTP_HOST"]): + return False referer_path = re.sub("http[s]{0,1}://.*?/", "/", referer).replace("/media", "") # Remove site address return referer_path.startswith("/" + site_address) diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index b1b8f9e2..f34c7443 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -530,7 +530,7 @@ class UiWebsocket(object): self.site.settings["size_limit"] = size_limit self.site.saveSettings() self.response(to, "Site size limit changed to %sMB" % size_limit) - self.site.download() + self.site.download(blind_includes=True) def actionServerUpdate(self, to): self.cmd("updating") diff --git a/src/Ui/media/Wrapper.coffee b/src/Ui/media/Wrapper.coffee index b79d1c34..1333bedb 100644 --- a/src/Ui/media/Wrapper.coffee +++ b/src/Ui/media/Wrapper.coffee @@ -290,9 +290,9 @@ class Wrapper @site_error = "No peers found" @loading.printLine "No peers found" - if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").indexOf("?") == -1 # First site info and mainpage + if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").replace("?wrapper=False", "").indexOf("?") == -1 # First site info and mainpage if site_info.size_limit*1.1 < site_info.next_size_limit # Need upgrade soon - @actionConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", => + @displayConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", => @ws.cmd "siteSetLimit", [site_info.next_size_limit], (res) => @notifications.add("size_limit", "done", res, 5000) return false @@ -323,7 +323,9 @@ class Wrapper @loading.printLine res @inner_loaded = false # Inner frame not loaded, just a 404 page displayed if reload - $("iframe").attr "src", $("iframe").attr("src")+"&"+(+new Date) # Reload iframe + src = $("iframe").attr("src") + $("iframe").attr "src", "" + $("iframe").attr "src", src return false diff --git a/src/Ui/media/Wrapper.css b/src/Ui/media/Wrapper.css index a3e66a4f..e2941308 100644 --- a/src/Ui/media/Wrapper.css +++ b/src/Ui/media/Wrapper.css @@ -19,7 +19,7 @@ a { color: black } .fixbutton { position: absolute; right: 35px; top: 15px; width: 40px; z-index: 999; - text-align: center; color: white; font-family: Consolas, Menlo, monospace; font-size: 25px; + text-align: center; color: white; font-family: Consolas, Monaco, monospace; font-size: 25px; } .fixbutton-bg { border-radius: 80px; background-color: rgba(180, 180, 180, 0.5); cursor: pointer; @@ -27,7 +27,7 @@ a { color: black } /*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); */ } .fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; perspective: 1000px; line-height: 0px; padding-top: 20px } -.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 48px; line-height: 32px } +.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px } .fixbutton-bg:hover { background-color: #AF3BFF } .fixbutton-bg:active { background-color: #9E2FEA; top: 1px; transition: none } diff --git a/src/Ui/media/all.css b/src/Ui/media/all.css index 81e8a74f..b59f612c 100644 --- a/src/Ui/media/all.css +++ b/src/Ui/media/all.css @@ -24,7 +24,7 @@ a { color: black } .fixbutton { position: absolute; right: 35px; top: 15px; width: 40px; z-index: 999; - text-align: center; color: white; font-family: Consolas, Menlo, monospace; font-size: 25px; + text-align: center; color: white; font-family: Consolas, Monaco, monospace; font-size: 25px; } .fixbutton-bg { -webkit-border-radius: 80px; -moz-border-radius: 80px; -o-border-radius: 80px; -ms-border-radius: 80px; border-radius: 80px ; background-color: rgba(180, 180, 180, 0.5); cursor: pointer; @@ -32,7 +32,7 @@ a { color: black } /*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* -webkit-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -moz-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -o-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -ms-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1) ; */ } .fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px } -.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 48px; line-height: 32px } +.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px } .fixbutton-bg:hover { background-color: #AF3BFF } .fixbutton-bg:active { background-color: #9E2FEA; top: 1px; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none } diff --git a/src/Ui/media/all.js b/src/Ui/media/all.js index 9e99b16e..1d589c4d 100644 --- a/src/Ui/media/all.js +++ b/src/Ui/media/all.js @@ -1140,9 +1140,9 @@ jQuery.extend( jQuery.easing, this.loading.printLine("No peers found"); } } - if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").indexOf("?") === -1) { + if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").replace("?wrapper=False", "").indexOf("?") === -1) { if (site_info.size_limit * 1.1 < site_info.next_size_limit) { - this.actionConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) { + this.displayConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) { return function() { _this.ws.cmd("siteSetLimit", [site_info.next_size_limit], function(res) { return _this.notifications.add("size_limit", "done", res, 5000); @@ -1184,10 +1184,13 @@ jQuery.extend( jQuery.easing, } this.ws.cmd("siteSetLimit", [size_limit], (function(_this) { return function(res) { + var src; _this.loading.printLine(res); _this.inner_loaded = false; if (reload) { - return $("iframe").attr("src", $("iframe").attr("src") + "&" + (+(new Date))); + src = $("iframe").attr("src"); + $("iframe").attr("src", ""); + return $("iframe").attr("src", src); } }; })(this)); @@ -1234,4 +1237,4 @@ jQuery.extend( jQuery.easing, window.wrapper = new Wrapper(ws_url); -}).call(this); \ No newline at end of file +}).call(this); diff --git a/src/Ui/template/wrapper.html b/src/Ui/template/wrapper.html index 5718a448..a52507fc 100644 --- a/src/Ui/template/wrapper.html +++ b/src/Ui/template/wrapper.html @@ -6,7 +6,7 @@ {title} - ZeroNet - + {meta_tags} @@ -57,7 +57,7 @@ permissions = {permissions} show_loadingscreen = {show_loadingscreen} server_url = '{server_url}' - + diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py index 905e4a72..23bbf459 100644 --- a/src/Worker/WorkerManager.py +++ b/src/Worker/WorkerManager.py @@ -78,7 +78,9 @@ class WorkerManager: return 9998 # index.html also important priority = task["priority"] if task["inner_path"].endswith(".js") or task["inner_path"].endswith(".css"): - priority += 1 # download js and css files first + priority += 2 # boost js and css files priority + elif task["inner_path"].endswith(".json"): + priority += 1 # boost json files priority return priority - task["workers_num"] # Prefer more priority and less workers # Returns the next free or less worked task diff --git a/src/main.py b/src/main.py index 7d3598a5..e01d05a7 100644 --- a/src/main.py +++ b/src/main.py @@ -65,6 +65,9 @@ config.parse() # Parse again to add plugin configuration options # Log current config logging.debug("Config: %s" % config) +# Use pure-python implementation of msgpack to save CPU +if config.msgpack_purepython: + os.environ["MSGPACK_PUREPYTHON"] = "True" # Socks Proxy monkey patch if config.proxy: