diff --git a/src/Config.py b/src/Config.py index 24dcbde7..47ac9919 100644 --- a/src/Config.py +++ b/src/Config.py @@ -8,7 +8,7 @@ class Config(object): def __init__(self, argv): self.version = "0.3.1" - self.rev = 330 + self.rev = 338 self.argv = argv self.action = None self.createParser() @@ -130,6 +130,8 @@ class Config(object): type='bool', choices=[True, False], default=False) self.parser.add_argument('--stream_downloads', help='Stream download directly to files (experimental)', type='bool', choices=[True, False], default=False) + self.parser.add_argument("--msgpack_purepython", help='Use less memory, but a bit more CPU power', + type='bool', choices=[True, False], default=False) self.parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript, metavar='executable_path') diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 8c166d4c..303faed0 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -237,6 +237,8 @@ class Connection(object): if read_bytes <= 0: break buff = self.sock.recv(16 * 1024) + if not buff: + break buff_len = len(buff) read_bytes -= buff_len file.write(buff) diff --git a/src/Db/Db.py b/src/Db/Db.py index 4f2034b9..34713c6e 100644 --- a/src/Db/Db.py +++ b/src/Db/Db.py @@ -4,9 +4,23 @@ import time import logging import re import os +import gevent from DbCursor import DbCursor +opened_dbs = [] + + +# Close idle databases to save some memory +def cleanup(): + while 1: + time.sleep(60 * 5) + for db in opened_dbs[:]: + if time.time() - db.last_query_time > 60 * 3: + db.close() + +gevent.spawn(cleanup) + class Db: @@ -22,8 +36,15 @@ class Db: self.collect_stats = False self.query_stats = {} self.db_keyvalues = {} + self.last_query_time = time.time() + + def __repr__(self): + return "" % self.db_path def connect(self): + if self not in opened_dbs: + opened_dbs.append(self) + self.log.debug("Connecting to %s (sqlite version: %s)..." % (self.db_path, sqlite3.version)) if not os.path.isdir(self.db_dir): # Directory not exist yet os.makedirs(self.db_dir) @@ -41,16 +62,21 @@ class Db: # Execute query using dbcursor def execute(self, query, params=None): + self.last_query_time = time.time() if not self.conn: self.connect() return self.cur.execute(query, params) def close(self): + if self in opened_dbs: + opened_dbs.remove(self) self.log.debug("Closing") if self.cur: self.cur.close() if self.conn: self.conn.close() + self.conn = None + self.cur = None # Gets a cursor object to database # Return: Cursor class diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 6e70114c..ba7089c4 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -73,7 +73,7 @@ class Peer(object): return self.connection def __str__(self): - return "Peer %-12s" % self.ip + return "Peer:%-12s" % self.ip def __repr__(self): return "<%s>" % self.__str__() @@ -126,7 +126,7 @@ class Peer(object): # Get a file content from peer def getFile(self, site, inner_path): # Use streamFile if client supports it - if config.stream_downloads and self.connection and self.connection.handshake["rev"] > 310: + if config.stream_downloads and self.connection and self.connection.handshake and self.connection.handshake["rev"] > 310: return self.streamFile(site, inner_path) location = 0 diff --git a/src/Site/Site.py b/src/Site/Site.py index 85cd07b8..7bc56e6a 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -158,8 +158,8 @@ class Site: # Download all files of the site @util.Noparallel(blocking=False) - def download(self, check_size=False): - self.log.debug("Start downloading...%s" % self.bad_files) + def download(self, check_size=False, blind_includes=False): + self.log.debug("Start downloading, bad_files: %s, check_size: %s, blind_includes: %s" % (self.bad_files, check_size, blind_includes)) gevent.spawn(self.announce) if check_size: # Check the size first valid = self.downloadContent(download_files=False) # Just download content.json files @@ -167,10 +167,14 @@ class Site: return False # Cant download content.jsons or size is not fits # Download everything - found = self.downloadContent("content.json") - self.checkModifications(0) # Download multiuser blind includes + valid = self.downloadContent("content.json") - return found + if valid and blind_includes: + self.checkModifications(0) # Download multiuser blind includes + + self.retryBadFiles() + + return valid # Update worker, try to find client that supports listModifications command def updater(self, peers_try, queried, since): @@ -189,7 +193,9 @@ class Site: queried.append(peer) for inner_path, modified in res["modified_files"].iteritems(): # Check if the peer has newer files than we content = self.content_manager.contents.get(inner_path) - if not content or modified > content["modified"]: # We dont have this file or we have older + if (not content or modified > content["modified"]) and inner_path not in self.bad_files: + self.log.debug("New modified file from %s: %s" % (peer, inner_path)) + # We dont have this file or we have older self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file gevent.spawn(self.downloadContent, inner_path) # Download the content.json + the changed files @@ -414,7 +420,7 @@ class Site: elif self.settings["serving"] is False: # Site not serving return False else: # Wait until file downloaded - self.bad_files[inner_path] = True # Mark as bad file + self.bad_files[inner_path] = self.bad_files.get(inner_path,0)+1 # Mark as bad file if not self.content_manager.contents.get("content.json"): # No content.json, download it first! self.log.debug("Need content.json first") gevent.spawn(self.announce) diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 34ba0562..1b61a5e3 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -78,9 +78,12 @@ class SiteManager(object): if not site.settings["serving"]: # Maybe it was deleted before site.settings["serving"] = True site.saveSettings() + if all_file: # Also download user files on first sync + site.download(blind_includes=True) + else: + if all_file: + site.download() - if all_file: - site.download() return site def delete(self, address): diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 7e37fcef..bf29472b 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -53,6 +53,10 @@ class UiRequest(object): # Media elif path.startswith("/uimedia/"): return self.actionUiMedia(path) + elif "/uimedia/" in path: + # uimedia within site dir (for chrome extension) + path = re.sub(".*?/uimedia/", "/uimedia/", path) + return self.actionUiMedia(path) elif path.startswith("/media"): return self.actionSiteMedia(path) # Websocket @@ -258,6 +262,8 @@ class UiRequest(object): # Returns if media request allowed from that referer def isMediaRequestAllowed(self, site_address, referer): + if not re.sub("^http[s]{0,1}://", "", referer).startswith(self.env["HTTP_HOST"]): + return False referer_path = re.sub("http[s]{0,1}://.*?/", "/", referer).replace("/media", "") # Remove site address return referer_path.startswith("/" + site_address) diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index b1b8f9e2..f34c7443 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -530,7 +530,7 @@ class UiWebsocket(object): self.site.settings["size_limit"] = size_limit self.site.saveSettings() self.response(to, "Site size limit changed to %sMB" % size_limit) - self.site.download() + self.site.download(blind_includes=True) def actionServerUpdate(self, to): self.cmd("updating") diff --git a/src/Ui/media/Wrapper.coffee b/src/Ui/media/Wrapper.coffee index b79d1c34..1333bedb 100644 --- a/src/Ui/media/Wrapper.coffee +++ b/src/Ui/media/Wrapper.coffee @@ -290,9 +290,9 @@ class Wrapper @site_error = "No peers found" @loading.printLine "No peers found" - if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").indexOf("?") == -1 # First site info and mainpage + if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").replace("?wrapper=False", "").indexOf("?") == -1 # First site info and mainpage if site_info.size_limit*1.1 < site_info.next_size_limit # Need upgrade soon - @actionConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", => + @displayConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", => @ws.cmd "siteSetLimit", [site_info.next_size_limit], (res) => @notifications.add("size_limit", "done", res, 5000) return false @@ -323,7 +323,9 @@ class Wrapper @loading.printLine res @inner_loaded = false # Inner frame not loaded, just a 404 page displayed if reload - $("iframe").attr "src", $("iframe").attr("src")+"&"+(+new Date) # Reload iframe + src = $("iframe").attr("src") + $("iframe").attr "src", "" + $("iframe").attr "src", src return false diff --git a/src/Ui/media/Wrapper.css b/src/Ui/media/Wrapper.css index a3e66a4f..e2941308 100644 --- a/src/Ui/media/Wrapper.css +++ b/src/Ui/media/Wrapper.css @@ -19,7 +19,7 @@ a { color: black } .fixbutton { position: absolute; right: 35px; top: 15px; width: 40px; z-index: 999; - text-align: center; color: white; font-family: Consolas, Menlo, monospace; font-size: 25px; + text-align: center; color: white; font-family: Consolas, Monaco, monospace; font-size: 25px; } .fixbutton-bg { border-radius: 80px; background-color: rgba(180, 180, 180, 0.5); cursor: pointer; @@ -27,7 +27,7 @@ a { color: black } /*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); */ } .fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; perspective: 1000px; line-height: 0px; padding-top: 20px } -.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 48px; line-height: 32px } +.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px } .fixbutton-bg:hover { background-color: #AF3BFF } .fixbutton-bg:active { background-color: #9E2FEA; top: 1px; transition: none } diff --git a/src/Ui/media/all.css b/src/Ui/media/all.css index 81e8a74f..b59f612c 100644 --- a/src/Ui/media/all.css +++ b/src/Ui/media/all.css @@ -24,7 +24,7 @@ a { color: black } .fixbutton { position: absolute; right: 35px; top: 15px; width: 40px; z-index: 999; - text-align: center; color: white; font-family: Consolas, Menlo, monospace; font-size: 25px; + text-align: center; color: white; font-family: Consolas, Monaco, monospace; font-size: 25px; } .fixbutton-bg { -webkit-border-radius: 80px; -moz-border-radius: 80px; -o-border-radius: 80px; -ms-border-radius: 80px; border-radius: 80px ; background-color: rgba(180, 180, 180, 0.5); cursor: pointer; @@ -32,7 +32,7 @@ a { color: black } /*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* -webkit-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -moz-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -o-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -ms-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1) ; */ } .fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px } -.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 48px; line-height: 32px } +.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px } .fixbutton-bg:hover { background-color: #AF3BFF } .fixbutton-bg:active { background-color: #9E2FEA; top: 1px; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none } diff --git a/src/Ui/media/all.js b/src/Ui/media/all.js index 9e99b16e..1d589c4d 100644 --- a/src/Ui/media/all.js +++ b/src/Ui/media/all.js @@ -1140,9 +1140,9 @@ jQuery.extend( jQuery.easing, this.loading.printLine("No peers found"); } } - if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").indexOf("?") === -1) { + if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").replace("?wrapper=False", "").indexOf("?") === -1) { if (site_info.size_limit * 1.1 < site_info.next_size_limit) { - this.actionConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) { + this.displayConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) { return function() { _this.ws.cmd("siteSetLimit", [site_info.next_size_limit], function(res) { return _this.notifications.add("size_limit", "done", res, 5000); @@ -1184,10 +1184,13 @@ jQuery.extend( jQuery.easing, } this.ws.cmd("siteSetLimit", [size_limit], (function(_this) { return function(res) { + var src; _this.loading.printLine(res); _this.inner_loaded = false; if (reload) { - return $("iframe").attr("src", $("iframe").attr("src") + "&" + (+(new Date))); + src = $("iframe").attr("src"); + $("iframe").attr("src", ""); + return $("iframe").attr("src", src); } }; })(this)); @@ -1234,4 +1237,4 @@ jQuery.extend( jQuery.easing, window.wrapper = new Wrapper(ws_url); -}).call(this); \ No newline at end of file +}).call(this); diff --git a/src/Ui/template/wrapper.html b/src/Ui/template/wrapper.html index 5718a448..a52507fc 100644 --- a/src/Ui/template/wrapper.html +++ b/src/Ui/template/wrapper.html @@ -6,7 +6,7 @@ {title} - ZeroNet - + {meta_tags} @@ -57,7 +57,7 @@ permissions = {permissions} show_loadingscreen = {show_loadingscreen} server_url = '{server_url}' - + diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py index 905e4a72..23bbf459 100644 --- a/src/Worker/WorkerManager.py +++ b/src/Worker/WorkerManager.py @@ -78,7 +78,9 @@ class WorkerManager: return 9998 # index.html also important priority = task["priority"] if task["inner_path"].endswith(".js") or task["inner_path"].endswith(".css"): - priority += 1 # download js and css files first + priority += 2 # boost js and css files priority + elif task["inner_path"].endswith(".json"): + priority += 1 # boost json files priority return priority - task["workers_num"] # Prefer more priority and less workers # Returns the next free or less worked task diff --git a/src/main.py b/src/main.py index 7d3598a5..e01d05a7 100644 --- a/src/main.py +++ b/src/main.py @@ -65,6 +65,9 @@ config.parse() # Parse again to add plugin configuration options # Log current config logging.debug("Config: %s" % config) +# Use pure-python implementation of msgpack to save CPU +if config.msgpack_purepython: + os.environ["MSGPACK_PUREPYTHON"] = "True" # Socks Proxy monkey patch if config.proxy: