Add option verify_files to Site.update() to allow the real content verification check, not just the simple file size-based one

Add more informative updateWebsocket() notification in Site.update() and SiteStorage.verifyFiles()

src/Site/Site.py

@@ -689,18 +689,28 @@ class Site(object):
     # Update content.json from peers and download changed files
     # Return: None
     @util.Noparallel()
-    def update(self, announce=False, check_files=False, since=None):
+    def update(self, announce=False, check_files=False, verify_files=False, since=None):
         self.content_manager.loadContent("content.json", load_includes=False)  # Reload content.json
         self.content_updated = None  # Reset content updated time
 
-        if check_files:
+        if verify_files:
+            check_files = True
+
+        self.updateWebsocket(updating=True)
+        if verify_files:
+            self.updateWebsocket(verifying=True)
+        elif check_files:
+            self.updateWebsocket(checking=True)
+
+        if verify_files:
+            self.storage.updateBadFiles(quick_check=False)
+        elif check_files:
             self.storage.updateBadFiles(quick_check=True)  # Quick check and mark bad files based on file size
 
         if not self.isServing():
+            self.updateWebsocket(updated=True)
             return False
 
-        self.updateWebsocket(updating=True)
-
         # Remove files that no longer in content.json
         self.checkBadFiles()
 
@@ -1573,6 +1583,7 @@ class Site(object):
             param = None
         for ws in self.websockets:
             ws.event("siteChanged", self, param)
+        time.sleep(0.001)
 
     def messageWebsocket(self, message, type="info", progress=None):
         for ws in self.websockets:

src/Site/SiteStorage.py

@@ -24,6 +24,25 @@ thread_pool_fs_read = ThreadPool.ThreadPool(config.threads_fs_read, name="FS rea
 thread_pool_fs_write = ThreadPool.ThreadPool(config.threads_fs_write, name="FS write")
 thread_pool_fs_batch = ThreadPool.ThreadPool(1, name="FS batch")
 
+class VerifyFiles_Notificator(object):
+    def __init__(self, site, quick_check):
+        self.site = site
+        self.quick_check = quick_check
+        self.scanned_files = 0
+        self.websocket_update_interval = 0.25
+        self.websocket_update_time = time.time()
+
+    def inc(self):
+        self.scanned_files += 1
+        if self.websocket_update_time + self.websocket_update_interval < time.time():
+            self.send()
+
+    def send(self):
+        self.websocket_update_time = time.time()
+        if self.quick_check:
+            self.site.updateWebsocket(checking=self.scanned_files)
+        else:
+            self.site.updateWebsocket(verifying=self.scanned_files)
 
 @PluginManager.acceptPlugins
 class SiteStorage(object):
@@ -427,11 +446,14 @@ class SiteStorage(object):
         i = 0
         self.log.debug("Verifing files...")
 
+        notificator = VerifyFiles_Notificator(self.site, quick_check)
+
         if not self.site.content_manager.contents.get("content.json"):  # No content.json, download it first
             self.log.debug("VerifyFile content.json not exists")
             self.site.needFile("content.json", update=True)  # Force update to fix corrupt file
             self.site.content_manager.loadContent()  # Reload content.json
-        for content_inner_path, content in list(self.site.content_manager.contents.items()):
+        for content_inner_path, content in self.site.content_manager.contents.iteritems():
+            notificator.inc()
             back["num_content"] += 1
             i += 1
             if i % 50 == 0:
@@ -442,6 +464,7 @@ class SiteStorage(object):
                 bad_files.append(content_inner_path)
 
             for file_relative_path in list(content.get("files", {}).keys()):
+                notificator.inc()
                 back["num_file"] += 1
                 file_inner_path = helper.getDirname(content_inner_path) + file_relative_path  # Relative to site dir
                 file_inner_path = file_inner_path.strip("/")  # Strip leading /
@@ -452,14 +475,19 @@ class SiteStorage(object):
                     bad_files.append(file_inner_path)
                     continue
 
+                err = None
+
                 if quick_check:
-                    ok = os.path.getsize(file_path) == content["files"][file_relative_path]["size"]
+                    file_size = os.path.getsize(file_path)
+                    expected_size = content["files"][file_relative_path]["size"]
+                    ok = file_size == expected_size
                     if not ok:
-                        err = "Invalid size"
+                        err = "Invalid size: %s - actual, %s - expected" % (file_size, expected_size)
                 else:
                     try:
                         ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb"))
-                    except Exception as err:
+                    except Exception as err2:
+                        err = err2
                         ok = False
 
                 if not ok:
@@ -472,6 +500,7 @@ class SiteStorage(object):
             optional_added = 0
             optional_removed = 0
             for file_relative_path in list(content.get("files_optional", {}).keys()):
+                notificator.inc()
                 back["num_optional"] += 1
                 file_node = content["files_optional"][file_relative_path]
                 file_inner_path = helper.getDirname(content_inner_path) + file_relative_path  # Relative to site dir
@@ -516,6 +545,8 @@ class SiteStorage(object):
                     (content_inner_path, len(content["files"]), quick_check, optional_added, optional_removed)
                 )
 
+        notificator.send()
+
         self.site.content_manager.contents.db.processDelayed()
         time.sleep(0.001)  # Context switch to avoid gevent hangs
         return back

src/Ui/UiWebsocket.py

@@ -912,9 +912,9 @@ class UiWebsocket(object):
         self.response(to, "ok")
 
     # Update site content.json
-    def actionSiteUpdate(self, to, address, check_files=False, since=None, announce=False):
+    def actionSiteUpdate(self, to, address, check_files=False, verify_files=False, since=None, announce=False):
         def updateThread():
-            site.update(announce=announce, check_files=check_files, since=since)
+            site.update(announce=announce, check_files=check_files, verify_files=verify_files, since=since)
             self.response(to, "Updated")
 
         site = self.server.sites.get(address)