oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix "changed size during iteration" in verifyFiles()

Browse source
This commit is contained in:
Vadim Ushakov 2021-10-22 14:38:05 +07:00
parent 19b840defd
commit 1ef129bdf9
1 changed files with 36 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
src/Site/SiteStorage.py
View file

@ -439,6 +439,8 @@ class SiteStorage(object):
return inner_path
# Verify all files sha512sum using content.json
# The result may not be accurate if self.site.isStopping().
# verifyFiles() return immediately in that case.
def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True):
bad_files = []
back = defaultdict(int)
@ -446,14 +448,44 @@ class SiteStorage(object):
i = 0
self.log.debug("Verifing files...")
notificator = VerifyFiles_Notificator(self.site, quick_check)
if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first
self.log.debug("VerifyFile content.json not exists")
self.site.needFile("content.json", update=True) # Force update to fix corrupt file
self.site.content_manager.loadContent() # Reload content.json
for content_inner_path, content in self.site.content_manager.contents.iteritems():
notificator.inc()
# Trying to read self.site.content_manager.contents without being stuck
# on reading the long file list and also without getting
# "RuntimeError: dictionary changed size during iteration"
# We can't use just list(iteritems()) since it loads all the contents files
# at once and gets unresponsive.
contents = {}
notificator = None
tries = 0
max_tries = 40
stop = False
while not stop:
try:
contents = {}
notificator = VerifyFiles_Notificator(self.site, quick_check)
for content_inner_path, content in self.site.content_manager.contents.iteritems():
notificator.inc()
contents[content_inner_path] = content
if self.site.isStopping():
stop = True
break
stop = True
except RuntimeError as err:
if "changed size during iteration" in str(err):
tries += 1
if tries >= max_tries:
self.log.info("contents.json file list changed during iteration. %s tries done. Giving up.", tries)
stop = True
self.log.info("contents.json file list changed during iteration. Trying again... (%s)", tries)
time.sleep(2 * tries)
else:
stop = True
for content_inner_path, content in contents.items():
back["num_content"] += 1
i += 1
if i % 50 == 0: