diff --git a/plugins/Bigfile/Test/TestBigfile.py b/plugins/Bigfile/Test/TestBigfile.py index e3e58cf2..9078b9bf 100644 --- a/plugins/Bigfile/Test/TestBigfile.py +++ b/plugins/Bigfile/Test/TestBigfile.py @@ -136,7 +136,7 @@ class TestBigfile: # Download site site_temp.download(blind_includes=True).join(timeout=5) - bad_files = site_temp.storage.verifyFiles(quick_check=True) + bad_files = site_temp.storage.verifyFiles(quick_check=True)["bad_files"] assert not bad_files # client_piecefield = peer_client.piecefields[file_info["sha512"]].tostring() diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index fc5d5252..37533d1a 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -4,6 +4,7 @@ import shutil import json import time import sys +from collections import defaultdict import sqlite3 import gevent.event @@ -376,25 +377,32 @@ class SiteStorage(object): # Verify all files sha512sum using content.json def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True): bad_files = [] + back = defaultdict(int) + back["bad_files"] = bad_files i = 0 + self.log.debug("Verifing files...") if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first self.log.debug("VerifyFile content.json not exists") self.site.needFile("content.json", update=True) # Force update to fix corrupt file self.site.content_manager.loadContent() # Reload content.json for content_inner_path, content in self.site.content_manager.contents.items(): + back["num_content"] += 1 i += 1 if i % 50 == 0: time.sleep(0.0001) # Context switch to avoid gevent hangs if not os.path.isfile(self.getPath(content_inner_path)): # Missing content.json file + back["num_content_missing"] += 1 self.log.debug("[MISSING] %s" % content_inner_path) bad_files.append(content_inner_path) for file_relative_path in content.get("files", {}).keys(): + back["num_file"] += 1 file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir file_inner_path = file_inner_path.strip("/") # Strip leading / file_path = self.getPath(file_inner_path) if not os.path.isfile(file_path): + back["num_file_missing"] += 1 self.log.debug("[MISSING] %s" % file_inner_path) bad_files.append(file_inner_path) continue @@ -410,6 +418,7 @@ class SiteStorage(object): ok = False if not ok: + back["num_file_invalid"] += 1 self.log.debug("[INVALID] %s: %s" % (file_inner_path, err)) if add_changed or content.get("cert_user_id"): # If updating own site only add changed user files bad_files.append(file_inner_path) @@ -418,6 +427,7 @@ class SiteStorage(object): optional_added = 0 optional_removed = 0 for file_relative_path in content.get("files_optional", {}).keys(): + back["num_optional"] += 1 file_node = content["files_optional"][file_relative_path] file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir file_inner_path = file_inner_path.strip("/") # Strip leading / @@ -455,16 +465,17 @@ class SiteStorage(object): ) time.sleep(0.0001) # Context switch to avoid gevent hangs - return bad_files + return back # Check and try to fix site files integrity def updateBadFiles(self, quick_check=True): s = time.time() - bad_files = self.verifyFiles( + res = self.verifyFiles( quick_check, add_optional=self.site.isDownloadable(""), add_changed=not self.site.settings.get("own") # Don't overwrite changed files if site owned ) + bad_files = res["bad_files"] self.site.bad_files = {} if bad_files: for bad_file in bad_files: diff --git a/src/Test/TestSite.py b/src/Test/TestSite.py index 71d1706f..368c6529 100644 --- a/src/Test/TestSite.py +++ b/src/Test/TestSite.py @@ -26,7 +26,7 @@ class TestSite: assert new_site.storage.isFile("index.html") assert new_site.storage.isFile("data/users/content.json") assert new_site.storage.isFile("data/zeroblog.db") - assert new_site.storage.verifyFiles() == [] # No bad files allowed + assert new_site.storage.verifyFiles()["bad_files"] == [] # No bad files allowed assert new_site.storage.query("SELECT * FROM keyvalue WHERE key = 'title'").fetchone()["value"] == "MyZeroBlog" # Test re-cloning (updating) diff --git a/src/Test/TestSiteDownload.py b/src/Test/TestSiteDownload.py index 13bf85c7..3c5a7531 100644 --- a/src/Test/TestSiteDownload.py +++ b/src/Test/TestSiteDownload.py @@ -48,7 +48,7 @@ class TestSiteDownload: assert "-default" in file_requests[-1] # Put default files for cloning to the end # Check files - bad_files = site_temp.storage.verifyFiles(quick_check=True) + bad_files = site_temp.storage.verifyFiles(quick_check=True)["bad_files"] # -1 because data/users/1J6... user has invalid cert assert len(site_temp.content_manager.contents) == len(site.content_manager.contents) - 1 @@ -72,7 +72,7 @@ class TestSiteDownload: # Download normally site_temp.addPeer("127.0.0.1", 1544) site_temp.download(blind_includes=True).join(timeout=5) - bad_files = site_temp.storage.verifyFiles(quick_check=True) + bad_files = site_temp.storage.verifyFiles(quick_check=True)["bad_files"] assert not bad_files assert "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json" in site_temp.content_manager.contents