Stats for total optional files and downloaded optional files in a site

src/Content/ContentManager.py

@@ -24,9 +24,10 @@ class ContentManager(object):
         self.hashfield = PeerHashfield()
         self.has_optional_files = False
 
+    # Load all content.json files
     def loadContents(self):
         if len(self.contents) == 0:
-            self.log.debug("Content db not initialized, load files from filesystem")
+            self.log.debug("ContentDb not initialized, load files from filesystem")
             self.loadContent(add_bad_files=False, delete_removed_files=False)
         self.site.settings["size"] = self.getTotalSize()
 
@@ -34,10 +35,10 @@ class ContentManager(object):
         if "hashfield" in self.site.settings.get("cache", {}):
             self.hashfield.fromstring(self.site.settings["cache"]["hashfield"].decode("base64"))
             del self.site.settings["cache"]["hashfield"]
-            self.has_optional_files = True
         elif self.contents.get("content.json") and self.getOptionalSize() > 0:
             self.site.storage.updateBadFiles()  # No hashfield cache created yet
-            self.has_optional_files = True
+        self.has_optional_files = bool(self.hashfield)
+
         self.contents.db.initSite(self.site)
 
     # Load content.json to self.content
@@ -99,7 +100,7 @@ class ContentManager(object):
                         changed.append(content_inner_dir + relative_path)  # Download new file
                     elif old_hash != new_hash and not self.site.settings.get("own"):
                         try:
-                            self.hashfield.removeHash(old_hash)
+                            self.optionalRemove(file_inner_path, old_hash, old_content["files_optional"][relative_path]["size"])
                             self.site.storage.delete(file_inner_path)
                             self.log.debug("Deleted changed optional file: %s" % file_inner_path)
                         except Exception, err:
@@ -120,12 +121,20 @@ class ContentManager(object):
                     **new_content.get("files_optional", {})
                 )
 
-                deleted = [content_inner_dir + key for key in old_files if key not in new_files]
+                deleted = [key for key in old_files if key not in new_files]
                 if deleted and not self.site.settings.get("own"):
                     # Deleting files that no longer in content.json
-                    for file_inner_path in deleted:
+                    for file_relative_path in deleted:
+                        file_inner_path = content_inner_dir + file_relative_path
                         try:
                             self.site.storage.delete(file_inner_path)
+
+                            # Check if the deleted file is optional
+                            if old_content.get("files_optional") and old_content["files_optional"].get(file_relative_path):
+                                old_hash = old_content["files_optional"][file_relative_path].get("sha512")
+                                if self.hashfield.hasHash(old_hash):
+                                    self.optionalRemove(file_inner_path, old_hash, old_content["files_optional"][file_relative_path]["size"])
+
                             self.log.debug("Deleted file: %s" % file_inner_path)
                         except Exception, err:
                             self.log.debug("Error deleting file %s: %s" % (file_inner_path, err))
@@ -195,6 +204,9 @@ class ContentManager(object):
             new_content["signs"] = None
             if "cert_sign" in new_content:
                 new_content["cert_sign"] = None
+
+            if new_content.get("files_optional"):
+                self.has_optional_files = True
             # Update the content
             self.contents[content_inner_path] = new_content
         except Exception, err:
@@ -460,12 +472,15 @@ class ContentManager(object):
             if ignored:  # Ignore content.json, defined regexp and files starting with .
                 self.log.info("- [SKIPPED] %s" % file_relative_path)
             else:
-                file_path = self.site.storage.getPath(dir_inner_path + "/" + file_relative_path)
+                file_inner_path = dir_inner_path + "/" + file_relative_path
+                file_path = self.site.storage.getPath(file_inner_path)
                 sha512sum = CryptHash.sha512sum(file_path)  # Calculate sha512 sum of file
                 if optional:
                     self.log.info("- [OPTIONAL] %s (SHA512: %s)" % (file_relative_path, sha512sum))
-                    files_optional_node[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
+                    file_size = os.path.getsize(file_path)
-                    self.hashfield.appendHash(sha512sum)
+                    files_optional_node[file_relative_path] = {"sha512": sha512sum, "size": file_size}
+                    if not self.hashfield.hasHash(sha512sum):
+                        self.optionalDownloaded(file_inner_path, sha512sum, file_size, own=True)
                 else:
                     self.log.info("- %s (SHA512: %s)" % (file_relative_path, sha512sum))
                     files_node[file_relative_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)}
@@ -636,12 +651,14 @@ class ContentManager(object):
         old_content = self.contents.get(inner_path)
         if old_content:
             old_content_size = len(json.dumps(old_content, indent=1)) + sum([file["size"] for file in old_content.get("files", {}).values()])
+            old_content_size_optional = sum([file["size"] for file in old_content.get("files_optional", {}).values()])
         else:
             old_content_size = 0
-
+            old_content_size_optional = 0
 
         content_size_optional = sum([file["size"] for file in content.get("files_optional", {}).values()])
         site_size = self.site.settings["size"] - old_content_size + content_size  # Site size without old content plus the new
+        site_size_optional = self.site.settings["size_optional"] - old_content_size_optional + content_size_optional  # Site size without old content plus the new
 
         site_size_limit = self.site.getSizeLimit() * 1024 * 1024
 
@@ -664,7 +681,8 @@ class ContentManager(object):
             return False
 
         if inner_path == "content.json":
-            self.site.settings["size"] = site_size  # Save to settings if larger
+            self.site.settings["size"] = site_size
+            self.site.settings["size_optional"] = site_size_optional
             return True  # Root content.json is passed
 
         # Load include details
@@ -704,7 +722,8 @@ class ContentManager(object):
             self.log.error("%s: Includes not allowed" % inner_path)
             return False  # Includes not allowed
 
-        self.site.settings["size"] = site_size  # Save to settings if larger
+        self.site.settings["size"] = site_size
+        self.site.settings["size_optional"] = site_size_optional
 
         return True  # All good
 
@@ -800,6 +819,26 @@ class ContentManager(object):
                 self.log.error("File not in content.json: %s" % inner_path)
                 return False
 
+    def optionalDownloaded(self, inner_path, hash, size=None, own=False):
+        if size is None:
+            size = self.site.storage.getSize(inner_path)
+        if type(hash) is int:
+            done = self.hashfield.appendHashId(hash)
+        else:
+            done = self.hashfield.appendHash(hash)
+        self.site.settings["optional_downloaded"] += size
+        return done
+
+    def optionalRemove(self, inner_path, hash, size=None):
+        if size is None:
+            size = self.site.storage.getSize(inner_path)
+        if type(hash) is int:
+            done = self.hashfield.removeHashId(hash)
+        else:
+            done = self.hashfield.removeHash(hash)
+        self.site.settings["optional_downloaded"] -= size
+        return done
+
 
 if __name__ == "__main__":
     def testSign():

src/Site/Site.py

@@ -82,13 +82,17 @@ class Site(object):
             self.settings = settings
             if "cache" not in settings:
                 settings["cache"] = {}
+            if "size_files_optional" not in settings:
+                settings["size_optional"] = 0
+            if "optional_downloaded" not in settings:
+                settings["optional_downloaded"] = 0
             self.bad_files = settings["cache"].get("bad_files", {})
             settings["cache"]["bad_files"] = {}
             # Reset tries
             for inner_path in self.bad_files:
                 self.bad_files[inner_path] = 1
         else:
-            self.settings = {"own": False, "serving": True, "permissions": [], "added": int(time.time())}  # Default
+            self.settings = {"own": False, "serving": True, "permissions": [], "added": int(time.time()), "optional_downloaded": 0, "size_optional": 0}  # Default
 
         # Add admin permissions to homepage
         if self.address == config.homepage and "ADMIN" not in self.settings["permissions"]: