From 1ef129bdf9193cbbc9aa1d0f595345e6453eff52 Mon Sep 17 00:00:00 2001
From: Vadim Ushakov <igeekless@gmail.com>
Date: Fri, 22 Oct 2021 14:38:05 +0700
Subject: [PATCH] Fix "changed size during iteration" in verifyFiles()
---
src/Site/SiteStorage.py | 40 ++++++++++++++++++++++++++++++++++++----
1 file changed, 36 insertions(+), 4 deletions(-)
diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py
index b89aedbf..97f720dc 100644
--- a/src/Site/SiteStorage.py
+++ b/src/Site/SiteStorage.py
@@ -439,6 +439,8 @@ class SiteStorage(object):
return inner_path
# Verify all files sha512sum using content.json
+ # The result may not be accurate if self.site.isStopping().
+ # verifyFiles() return immediately in that case.
def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True):
bad_files = []
back = defaultdict(int)
@@ -446,14 +448,44 @@ class SiteStorage(object):
i = 0
self.log.debug("Verifing files...")
- notificator = VerifyFiles_Notificator(self.site, quick_check)
-
if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first
self.log.debug("VerifyFile content.json not exists")
self.site.needFile("content.json", update=True) # Force update to fix corrupt file
self.site.content_manager.loadContent() # Reload content.json
- for content_inner_path, content in self.site.content_manager.contents.iteritems():
- notificator.inc()
+
+ # Trying to read self.site.content_manager.contents without being stuck
+ # on reading the long file list and also without getting
+ # "RuntimeError: dictionary changed size during iteration"
+ # We can't use just list(iteritems()) since it loads all the contents files
+ # at once and gets unresponsive.
+ contents = {}
+ notificator = None
+ tries = 0
+ max_tries = 40
+ stop = False
+ while not stop:
+ try:
+ contents = {}
+ notificator = VerifyFiles_Notificator(self.site, quick_check)
+ for content_inner_path, content in self.site.content_manager.contents.iteritems():
+ notificator.inc()
+ contents[content_inner_path] = content
+ if self.site.isStopping():
+ stop = True
+ break
+ stop = True
+ except RuntimeError as err:
+ if "changed size during iteration" in str(err):
+ tries += 1
+ if tries >= max_tries:
+ self.log.info("contents.json file list changed during iteration. %s tries done. Giving up.", tries)
+ stop = True
+ self.log.info("contents.json file list changed during iteration. Trying again... (%s)", tries)
+ time.sleep(2 * tries)
+ else:
+ stop = True
+
+ for content_inner_path, content in contents.items():
back["num_content"] += 1
i += 1
if i % 50 == 0: