From 0459c75dc0262a7bbb033326690cca643b9f5877 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Thu, 29 Mar 2018 03:19:26 +0200 Subject: [PATCH] Make file verification process handle correctly different files with same has_id by using new isDownloaded function --- src/Site/SiteStorage.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index 37533d1a..49297f79 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -432,9 +432,12 @@ class SiteStorage(object): file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir file_inner_path = file_inner_path.strip("/") # Strip leading / file_path = self.getPath(file_inner_path) + hash_id = self.site.content_manager.hashfield.getHashId(file_node["sha512"]) if not os.path.isfile(file_path): - if self.site.content_manager.hashfield.hasHash(file_node["sha512"]): - self.site.content_manager.optionalRemove(file_inner_path, file_node["sha512"], file_node["size"]) + if self.site.content_manager.isDownloaded(file_inner_path, hash_id): + back["num_optional_removed"] += 1 + self.log.debug("[OPTIONAL REMOVED] %s" % file_inner_path) + self.site.content_manager.optionalRemoved(file_inner_path, hash_id, file_node["size"]) if add_optional: bad_files.append(file_inner_path) continue @@ -448,12 +451,15 @@ class SiteStorage(object): ok = False if ok: - if not self.site.content_manager.hashfield.hasHash(file_node["sha512"]): - self.site.content_manager.optionalDownloaded(file_inner_path, file_node["sha512"], file_node["size"]) + if not self.site.content_manager.isDownloaded(file_inner_path, hash_id): + back["num_optional_added"] += 1 + self.site.content_manager.optionalDownloaded(file_inner_path, hash_id, file_node["size"]) optional_added += 1 + self.log.debug("[OPTIONAL FOUND] %s" % file_inner_path) else: - if self.site.content_manager.hashfield.hasHash(file_node["sha512"]): - self.site.content_manager.optionalRemove(file_inner_path, file_node["sha512"], file_node["size"]) + if self.site.content_manager.isDownloaded(file_inner_path, hash_id): + back["num_optional_removed"] += 1 + self.site.content_manager.optionalRemoved(file_inner_path, hash_id, file_node["size"]) optional_removed += 1 bad_files.append(file_inner_path) self.log.debug("[OPTIONAL CHANGED] %s" % file_inner_path) @@ -464,6 +470,7 @@ class SiteStorage(object): (content_inner_path, len(content["files"]), quick_check, optional_added, optional_removed) ) + self.site.content_manager.contents.db.processDelayed() time.sleep(0.0001) # Context switch to avoid gevent hangs return back