Rev409, Delete files removed from content.json, Start download user files right after content.json downloaded, New API command: fileDelete, Better notification input auto focus

src/Config.py

@@ -8,7 +8,7 @@ class Config(object):
 
     def __init__(self, argv):
         self.version = "0.3.2"
-        self.rev = 399
+        self.rev = 409
         self.argv = argv
         self.action = None
         self.createParser()

src/Content/ContentManager.py

@@ -21,27 +21,29 @@ class ContentManager(object):
         self.site.settings["size"] = self.getTotalSize()
 
     # Load content.json to self.content
-    # Return: Changed files ["index.html", "data/messages.json"]
+    # Return: Changed files ["index.html", "data/messages.json"], Deleted files ["old.jpg"]
-    def loadContent(self, content_inner_path="content.json", add_bad_files=True, load_includes=True):
+    def loadContent(self, content_inner_path="content.json", add_bad_files=True, delete_removed_files=True, load_includes=True):
         content_inner_path = content_inner_path.strip("/")  # Remove / from begning
         old_content = self.contents.get(content_inner_path)
         content_path = self.site.storage.getPath(content_inner_path)
-        content_path_dir = self.toDir(self.site.storage.getPath(content_inner_path))
+        content_dir = self.toDir(self.site.storage.getPath(content_inner_path))
-        content_dir = self.toDir(content_inner_path)
+        content_inner_dir = self.toDir(content_inner_path)
 
         if os.path.isfile(content_path):
             try:
                 new_content = json.load(open(content_path))
             except Exception, err:
                 self.log.error("%s load error: %s" % (content_path, Debug.formatException(err)))
-                return False
+                return [], []
         else:
             self.log.error("Content.json not exist: %s" % content_path)
-            return False  # Content.json not exist
+            return [], []  # Content.json not exist
 
         try:
             # Get the files where the sha512 changed
             changed = []
+            deleted = []
+            # Check changed
             for relative_path, info in new_content.get("files", {}).items():
                 if "sha512" in info:
                     hash_type = "sha512"
@@ -54,46 +56,66 @@ class ContentManager(object):
                 else:  # The file is not in the old content
                     old_hash = None
                 if old_hash != new_hash:
-                    changed.append(content_dir + relative_path)
+                    changed.append(content_inner_dir + relative_path)
+
+            # Check deleted
+            if old_content:
+                deleted = [
+                    content_inner_dir + key for key in old_content.get("files", {}) if key not in new_content.get("files", {})
+                ]
+                if deleted:
+                    # Deleting files that no longer in content.json
+                    for file_inner_path in deleted:
+                        self.log.debug("Deleting file: %s" % file_inner_path)
+                        self.site.storage.delete(file_inner_path)
 
             # Load includes
             if load_includes and "includes" in new_content:
                 for relative_path, info in new_content["includes"].items():
-                    include_inner_path = content_dir + relative_path
+                    include_inner_path = content_inner_dir + relative_path
                     if self.site.storage.isFile(include_inner_path):  # Content.json exists, load it
-                        success = self.loadContent(include_inner_path, add_bad_files=add_bad_files)
+                        include_changed, include_deleted = self.loadContent(
-                        if success:
+                            include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files
-                            changed += success  # Add changed files
+                        )
+                        if include_changed:
+                            changed += include_changed  # Add changed files
+                        if include_deleted:
+                            deleted += include_deleted  # Add changed files
                     else:  # Content.json not exist, add to changed files
                         self.log.debug("Missing include: %s" % include_inner_path)
                         changed += [include_inner_path]
 
             # Load blind user includes (all subdir)
             if load_includes and "user_contents" in new_content:
-                for relative_dir in os.listdir(content_path_dir):
+                for relative_dir in os.listdir(content_dir):
-                    include_inner_path = content_dir + relative_dir + "/content.json"
+                    include_inner_path = content_inner_dir + relative_dir + "/content.json"
                     if not self.site.storage.isFile(include_inner_path):
                         continue  # Content.json not exist
-                    success = self.loadContent(include_inner_path, add_bad_files=add_bad_files, load_includes=False)
+                    include_changed, include_deleted = self.loadContent(
-                    if success:
+                        include_inner_path, add_bad_files=add_bad_files, delete_removed_files=delete_removed_files,
-                        changed += success  # Add changed files
+                        load_includes=False
+                    )
+                    if include_changed:
+                        changed += include_changed  # Add changed files
+                    if include_deleted:
+                        deleted += include_deleted  # Add changed files
 
             # Update the content
             self.contents[content_inner_path] = new_content
         except Exception, err:
             self.log.error("Content.json parse error: %s" % Debug.formatException(err))
-            return False  # Content.json parse error
+            return [], []  # Content.json parse error
 
         # Add changed files to bad files
         if add_bad_files:
             for inner_path in changed:
-                self.site.bad_files[inner_path] = True
+                self.site.bad_files[inner_path] = self.site.bad_files.get(inner_path, 0) + 1
 
         if new_content["modified"] > self.site.settings.get("modified", 0):
             # Dont store modifications in the far future (more than 10 minute)
             self.site.settings["modified"] = min(time.time() + 60 * 10, new_content["modified"])
 
-        return changed
+        return changed, deleted
 
     # Get total size of site
     # Return: 32819 (size of files in kb)

src/File/FileRequest.py

@@ -84,8 +84,8 @@ class FileRequest(object):
                 "Someone trying to push a file to own site %s, reload local %s first" %
                 (site.address, params["inner_path"])
             )
-            changed = site.content_manager.loadContent(params["inner_path"], add_bad_files=False)
+            changed, deleted = site.content_manager.loadContent(params["inner_path"], add_bad_files=False)
-            if changed:  # Content.json changed locally
+            if changed or deleted:  # Content.json changed locally
                 site.settings["size"] = site.content_manager.getTotalSize()  # Update site size
         buff = StringIO(params["body"])
         valid = site.content_manager.verifyFile(params["inner_path"], buff)

src/Site/Site.py

@@ -105,7 +105,7 @@ class Site:
         return 999999
 
     # Download all file from content.json
-    def downloadContent(self, inner_path, download_files=True, peer=None):
+    def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False):
         s = time.time()
         self.log.debug("Downloading %s..." % inner_path)
         found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
@@ -114,7 +114,7 @@ class Site:
             return False  # Could not download content.json
 
         self.log.debug("Got %s" % inner_path)
-        changed = self.content_manager.loadContent(inner_path, load_includes=False)
+        changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False)
 
         # Start download files
         file_threads = []
@@ -137,6 +137,9 @@ class Site:
         gevent.joinall(include_threads)
         self.log.debug("%s: Includes download ended" % inner_path)
 
+        if check_modifications:  # Check if every file is up-to-date
+            self.checkModifications(0)
+
         self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
         gevent.joinall(file_threads)
         self.log.debug("%s: DownloadContent ended in %.2fs" % (inner_path, time.time() - s))
@@ -168,10 +171,7 @@ class Site:
                 return False  # Cant download content.jsons or size is not fits
 
         # Download everything
-        valid = self.downloadContent("content.json")
+        valid = self.downloadContent("content.json", check_modifications=blind_includes)
-
-        if valid and blind_includes:
-            self.checkModifications(0)  # Download multiuser blind includes
 
         self.retryBadFiles()
 
@@ -246,10 +246,7 @@ class Site:
         if not self.settings["own"]:
             self.storage.checkFiles(quick_check=True)  # Quick check files based on file size
 
-        changed = self.content_manager.loadContent("content.json")
+        changed, deleted = self.content_manager.loadContent("content.json")
-        if changed:
-            for changed_file in changed:
-                self.bad_files[changed_file] = self.bad_files.get(changed_file, 0) + 1
 
         if self.bad_files:
             self.download()
@@ -376,7 +373,9 @@ class Site:
             if address_index:
                 content_json["address_index"] = address_index  # Site owner's BIP32 index
             new_site.storage.writeJson("content.json", content_json)
-            new_site.content_manager.loadContent("content.json", add_bad_files=False, load_includes=False)
+            new_site.content_manager.loadContent(
+                "content.json", add_bad_files=False, delete_removed_files=False, load_includes=False
+            )
 
         # Copy files
         for content_inner_path, content in self.content_manager.contents.items():
@@ -411,7 +410,8 @@ class Site:
                     if file_path_dest.endswith("/content.json"):
                         new_site.storage.onUpdated(file_inner_path.replace("-default", ""))
                         new_site.content_manager.loadContent(
-                            file_inner_path.replace("-default", ""), add_bad_files=False, load_includes=False
+                            file_inner_path.replace("-default", ""), add_bad_files=False,
+                            delete_removed_files=False, load_includes=False
                         )
                         if privatekey:
                             new_site.content_manager.sign(file_inner_path.replace("-default", ""), privatekey)
@@ -607,7 +607,8 @@ class Site:
             threads.append(thread)
             thread.address = address
             thread.protocol = protocol
-            if len(threads) > num: break  # Announce limit
+            if len(threads) > num:  # Announce limit
+                break
 
         gevent.joinall(threads)  # Wait for announce finish
 

src/Site/SiteStorage.py

@@ -151,6 +151,11 @@ class SiteStorage:
         del content
         self.onUpdated(inner_path)
 
+    # Remove file from filesystem
+    def delete(self, inner_path):
+        file_path = self.getPath(inner_path)
+        os.unlink(file_path)
+
     # Site content updated
     def onUpdated(self, inner_path):
         file_path = self.getPath(inner_path)

src/Ui/UiWebsocket.py

@@ -343,6 +343,25 @@ class UiWebsocket(object):
             if ws != self:
                 ws.event("siteChanged", self.site, {"event": ["file_done", inner_path]})
 
+    def actionFileDelete(self, to, inner_path):
+        if (
+            not self.site.settings["own"] and
+            self.user.getAuthAddress(self.site.address) not in self.site.content_manager.getValidSigners(inner_path)
+        ):
+            return self.response(to, "Forbidden, you can only modify your own files")
+
+        try:
+            self.site.storage.delete(inner_path)
+        except Exception, err:
+            return self.response(to, "Delete error: %s" % err)
+
+        self.response(to, "ok")
+
+        # Send sitechanged to other local users
+        for ws in self.site.websockets:
+            if ws != self:
+                ws.event("siteChanged", self.site, {"event": ["file_deleted", inner_path]})
+
     # Find data in json files
     def actionFileQuery(self, to, dir_inner_path, query):
         # s = time.time()

src/Ui/media/Wrapper.coffee

@@ -117,7 +117,8 @@ class Wrapper
 		body.append(button)
 		@notifications.add("notification-#{caption}", "ask", body)
 
-		setTimeout (-> button.focus() ), 1500
+		button.focus()
+		$(".notification").scrollLeft(0)
 
 
 	actionConfirm: (message, cb=false) ->
@@ -145,7 +146,8 @@ class Wrapper
 
 		@notifications.add("notification-#{message.id}", "ask", body)
 
-		setTimeout (-> input.focus() ), 1500
+		input.focus()
+		$(".notification").scrollLeft(0)
 
 
 	actionPrompt: (message) ->

src/Ui/media/all.js

@@ -898,9 +898,8 @@ jQuery.extend( jQuery.easing,
       button.on("click", cb);
       body.append(button);
       this.notifications.add("notification-" + caption, "ask", body);
-      return setTimeout((function() {
+      button.focus();
-        return button.focus();
+      return $(".notification").scrollLeft(0);
-      }), 1500);
     };
 
     Wrapper.prototype.actionConfirm = function(message, cb) {
@@ -947,9 +946,8 @@ jQuery.extend( jQuery.easing,
       })(this));
       body.append(button);
       this.notifications.add("notification-" + message.id, "ask", body);
-      return setTimeout((function() {
+      input.focus();
-        return input.focus();
+      return $(".notification").scrollLeft(0);
-      }), 1500);
     };
 
     Wrapper.prototype.actionPrompt = function(message) {