From 917393c0227b1e87d9206b25dbb17941d7692fac Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Wed, 16 Sep 2015 00:01:23 +0200 Subject: [PATCH] Rev409, Delete files removed from content.json, Start download user files right after content.json downloaded, New API command: fileDelete, Better notification input auto focus --- src/Config.py | 2 +- src/Content/ContentManager.py | 60 ++++++++++++++++++++++++----------- src/File/FileRequest.py | 4 +-- src/Site/Site.py | 27 ++++++++-------- src/Site/SiteStorage.py | 5 +++ src/Ui/UiWebsocket.py | 19 +++++++++++ src/Ui/media/Wrapper.coffee | 6 ++-- src/Ui/media/all.js | 10 +++--- 8 files changed, 90 insertions(+), 43 deletions(-) diff --git a/src/Config.py b/src/Config.py index 7b060a9b..85ab7192 100644 --- a/src/Config.py +++ b/src/Config.py @@ -8,7 +8,7 @@ class Config(object): def __init__(self, argv): self.version = "0.3.2" - self.rev = 399 + self.rev = 409 self.argv = argv self.action = None self.createParser() diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index 9b4b0266..ef8a062e 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -21,27 +21,29 @@ class ContentManager(object): self.site.settings["size"] = self.getTotalSize() # Load content.json to self.content - # Return: Changed files ["index.html", "data/messages.json"] - def loadContent(self, content_inner_path="content.json", add_bad_files=True, load_includes=True): + # Return: Changed files ["index.html", "data/messages.json"], Deleted files ["old.jpg"] + def loadContent(self, content_inner_path="content.json", add_bad_files=True, delete_removed_files=True, load_includes=True): content_inner_path = content_inner_path.strip("/") # Remove / from begning old_content = self.contents.get(content_inner_path) content_path = self.site.storage.getPath(content_inner_path) - content_path_dir = self.toDir(self.site.storage.getPath(content_inner_path)) - content_dir = self.toDir(content_inner_path) + content_dir = self.toDir(self.site.storage.getPath(content_inner_path)) + content_inner_dir = self.toDir(content_inner_path) if os.path.isfile(content_path): try: new_content = json.load(open(content_path)) except Exception, err: self.log.error("%s load error: %s" % (content_path, Debug.formatException(err))) - return False + return [], [] else: self.log.error("Content.json not exist: %s" % content_path) - return False # Content.json not exist + return [], [] # Content.json not exist try: # Get the files where the sha512 changed changed = [] + deleted = [] + # Check changed for relative_path, info in new_content.get("files", {}).items(): if "sha512" in info: hash_type = "sha512" @@ -54,46 +56,66 @@ class ContentManager(object): else: # The file is not in the old content old_hash = None if old_hash != new_hash: - changed.append(content_dir + relative_path) + changed.append(content_inner_dir + relative_path) + + # Check deleted + if old_content: + deleted = [ + content_inner_dir + key for key in old_content.get("files", {}) if key not in new_content.get("files", {}) + ] + if deleted: + # Deleting files that no longer in content.json + for file_inner_path in deleted: + self.log.debug("Deleting file: %s" % file_inner_path) + self.site.storage.delete(file_inner_path) # Load includes if load_includes and "includes" in new_content: for relative_path, info in new_content["includes"].items(): - include_inner_path = content_dir + relative_path + include_inner_path = content_inner_dir + relative_path if self.site.storage.isFile(include_inner_path): # Content.json exists, load it - success = self.loadContent(include_inner_path, add_bad_files=add_bad_files) - if success: - changed += success # Add changed files + include_changed, include_deleted = self.loadContent( + include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files + ) + if include_changed: + changed += include_changed # Add changed files + if include_deleted: + deleted += include_deleted # Add changed files else: # Content.json not exist, add to changed files self.log.debug("Missing include: %s" % include_inner_path) changed += [include_inner_path] # Load blind user includes (all subdir) if load_includes and "user_contents" in new_content: - for relative_dir in os.listdir(content_path_dir): - include_inner_path = content_dir + relative_dir + "/content.json" + for relative_dir in os.listdir(content_dir): + include_inner_path = content_inner_dir + relative_dir + "/content.json" if not self.site.storage.isFile(include_inner_path): continue # Content.json not exist - success = self.loadContent(include_inner_path, add_bad_files=add_bad_files, load_includes=False) - if success: - changed += success # Add changed files + include_changed, include_deleted = self.loadContent( + include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files, + load_includes=False + ) + if include_changed: + changed += include_changed # Add changed files + if include_deleted: + deleted += include_deleted # Add changed files # Update the content self.contents[content_inner_path] = new_content except Exception, err: self.log.error("Content.json parse error: %s" % Debug.formatException(err)) - return False # Content.json parse error + return [], [] # Content.json parse error # Add changed files to bad files if add_bad_files: for inner_path in changed: - self.site.bad_files[inner_path] = True + self.site.bad_files[inner_path] = self.site.bad_files.get(inner_path, 0) + 1 if new_content["modified"] > self.site.settings.get("modified", 0): # Dont store modifications in the far future (more than 10 minute) self.site.settings["modified"] = min(time.time() + 60 * 10, new_content["modified"]) - return changed + return changed, deleted # Get total size of site # Return: 32819 (size of files in kb) diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index 116f7d18..96ed4c2f 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -84,8 +84,8 @@ class FileRequest(object): "Someone trying to push a file to own site %s, reload local %s first" % (site.address, params["inner_path"]) ) - changed = site.content_manager.loadContent(params["inner_path"], add_bad_files=False) - if changed: # Content.json changed locally + changed, deleted = site.content_manager.loadContent(params["inner_path"], add_bad_files=False) + if changed or deleted: # Content.json changed locally site.settings["size"] = site.content_manager.getTotalSize() # Update site size buff = StringIO(params["body"]) valid = site.content_manager.verifyFile(params["inner_path"], buff) diff --git a/src/Site/Site.py b/src/Site/Site.py index ab6ba3e6..f7c31d91 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -105,7 +105,7 @@ class Site: return 999999 # Download all file from content.json - def downloadContent(self, inner_path, download_files=True, peer=None): + def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False): s = time.time() self.log.debug("Downloading %s..." % inner_path) found = self.needFile(inner_path, update=self.bad_files.get(inner_path)) @@ -114,7 +114,7 @@ class Site: return False # Could not download content.json self.log.debug("Got %s" % inner_path) - changed = self.content_manager.loadContent(inner_path, load_includes=False) + changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False) # Start download files file_threads = [] @@ -137,6 +137,9 @@ class Site: gevent.joinall(include_threads) self.log.debug("%s: Includes download ended" % inner_path) + if check_modifications: # Check if every file is up-to-date + self.checkModifications(0) + self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed))) gevent.joinall(file_threads) self.log.debug("%s: DownloadContent ended in %.2fs" % (inner_path, time.time() - s)) @@ -168,10 +171,7 @@ class Site: return False # Cant download content.jsons or size is not fits # Download everything - valid = self.downloadContent("content.json") - - if valid and blind_includes: - self.checkModifications(0) # Download multiuser blind includes + valid = self.downloadContent("content.json", check_modifications=blind_includes) self.retryBadFiles() @@ -246,10 +246,7 @@ class Site: if not self.settings["own"]: self.storage.checkFiles(quick_check=True) # Quick check files based on file size - changed = self.content_manager.loadContent("content.json") - if changed: - for changed_file in changed: - self.bad_files[changed_file] = self.bad_files.get(changed_file, 0) + 1 + changed, deleted = self.content_manager.loadContent("content.json") if self.bad_files: self.download() @@ -376,7 +373,9 @@ class Site: if address_index: content_json["address_index"] = address_index # Site owner's BIP32 index new_site.storage.writeJson("content.json", content_json) - new_site.content_manager.loadContent("content.json", add_bad_files=False, load_includes=False) + new_site.content_manager.loadContent( + "content.json", add_bad_files=False, delete_removed_files=False, load_includes=False + ) # Copy files for content_inner_path, content in self.content_manager.contents.items(): @@ -411,7 +410,8 @@ class Site: if file_path_dest.endswith("/content.json"): new_site.storage.onUpdated(file_inner_path.replace("-default", "")) new_site.content_manager.loadContent( - file_inner_path.replace("-default", ""), add_bad_files=False, load_includes=False + file_inner_path.replace("-default", ""), add_bad_files=False, + delete_removed_files=False, load_includes=False ) if privatekey: new_site.content_manager.sign(file_inner_path.replace("-default", ""), privatekey) @@ -607,7 +607,8 @@ class Site: threads.append(thread) thread.address = address thread.protocol = protocol - if len(threads) > num: break # Announce limit + if len(threads) > num: # Announce limit + break gevent.joinall(threads) # Wait for announce finish diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index 4292267a..2d7f0193 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -151,6 +151,11 @@ class SiteStorage: del content self.onUpdated(inner_path) + # Remove file from filesystem + def delete(self, inner_path): + file_path = self.getPath(inner_path) + os.unlink(file_path) + # Site content updated def onUpdated(self, inner_path): file_path = self.getPath(inner_path) diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index e8cba7c0..f58bdb53 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -343,6 +343,25 @@ class UiWebsocket(object): if ws != self: ws.event("siteChanged", self.site, {"event": ["file_done", inner_path]}) + def actionFileDelete(self, to, inner_path): + if ( + not self.site.settings["own"] and + self.user.getAuthAddress(self.site.address) not in self.site.content_manager.getValidSigners(inner_path) + ): + return self.response(to, "Forbidden, you can only modify your own files") + + try: + self.site.storage.delete(inner_path) + except Exception, err: + return self.response(to, "Delete error: %s" % err) + + self.response(to, "ok") + + # Send sitechanged to other local users + for ws in self.site.websockets: + if ws != self: + ws.event("siteChanged", self.site, {"event": ["file_deleted", inner_path]}) + # Find data in json files def actionFileQuery(self, to, dir_inner_path, query): # s = time.time() diff --git a/src/Ui/media/Wrapper.coffee b/src/Ui/media/Wrapper.coffee index 6df4c371..e811dee6 100644 --- a/src/Ui/media/Wrapper.coffee +++ b/src/Ui/media/Wrapper.coffee @@ -117,7 +117,8 @@ class Wrapper body.append(button) @notifications.add("notification-#{caption}", "ask", body) - setTimeout (-> button.focus() ), 1500 + button.focus() + $(".notification").scrollLeft(0) actionConfirm: (message, cb=false) -> @@ -145,7 +146,8 @@ class Wrapper @notifications.add("notification-#{message.id}", "ask", body) - setTimeout (-> input.focus() ), 1500 + input.focus() + $(".notification").scrollLeft(0) actionPrompt: (message) -> diff --git a/src/Ui/media/all.js b/src/Ui/media/all.js index 36ee7588..f4caf715 100644 --- a/src/Ui/media/all.js +++ b/src/Ui/media/all.js @@ -898,9 +898,8 @@ jQuery.extend( jQuery.easing, button.on("click", cb); body.append(button); this.notifications.add("notification-" + caption, "ask", body); - return setTimeout((function() { - return button.focus(); - }), 1500); + button.focus(); + return $(".notification").scrollLeft(0); }; Wrapper.prototype.actionConfirm = function(message, cb) { @@ -947,9 +946,8 @@ jQuery.extend( jQuery.easing, })(this)); body.append(button); this.notifications.add("notification-" + message.id, "ask", body); - return setTimeout((function() { - return input.focus(); - }), 1500); + input.focus(); + return $(".notification").scrollLeft(0); }; Wrapper.prototype.actionPrompt = function(message) {