oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Move bad file checking to separate function, root content.json does not requires file_info

Browse source
This commit is contained in:
shortcutme 2017-11-19 18:21:10 +01:00
parent 972bef4f7d
commit 51f1781df4
No known key found for this signature in database
GPG key ID: 5B63BAE6CB9613AE
1 changed files with 16 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
src/Site/Site.py
View file

@ -260,24 +260,14 @@ class Site(object):
# Retry download bad files # Retry download bad files
def retryBadFiles(self, force=False): def retryBadFiles(self, force=False):
self.checkBadFiles()
self.log.debug("Retry %s bad files" % len(self.bad_files)) self.log.debug("Retry %s bad files" % len(self.bad_files))
content_inner_paths = [] content_inner_paths = []
file_inner_paths = [] file_inner_paths = []
for bad_file, tries in self.bad_files.items(): for bad_file, tries in self.bad_files.items():
if force or random.randint(0, min(40, tries)) < 4: # Larger number tries = less likely to check every 15min if force or random.randint(0, min(40, tries)) < 4: # Larger number tries = less likely to check every 15min
# Skip files without info
file_info = self.content_manager.getFileInfo(bad_file)
if bad_file.endswith("content.json"):
if file_info is False:
del self.bad_files[bad_file]
self.log.debug("No info for file: %s, removing from bad_files" % bad_file)
continue
else:
if file_info is False or not file_info.get("size"):
del self.bad_files[bad_file]
self.log.debug("No info for file: %s, removing from bad_files" % bad_file)
continue
if bad_file.endswith("content.json"): if bad_file.endswith("content.json"):
content_inner_paths.append(bad_file) content_inner_paths.append(bad_file)
else: else:
@ -289,6 +279,18 @@ class Site(object):
if file_inner_paths: if file_inner_paths:
self.pooledDownloadFile(file_inner_paths, only_if_bad=True) self.pooledDownloadFile(file_inner_paths, only_if_bad=True)
def checkBadFiles(self):
for bad_file in self.bad_files.keys():
file_info = self.content_manager.getFileInfo(bad_file)
if bad_file.endswith("content.json"):
if file_info is False and bad_file != "content.json":
del self.bad_files[bad_file]
self.log.debug("No info for file: %s, removing from bad_files" % bad_file)
else:
if file_info is False or not file_info.get("size"):
del self.bad_files[bad_file]
self.log.debug("No info or size for file: %s, removing from bad_files" % bad_file)
# Download all files of the site # Download all files of the site
@util.Noparallel(blocking=False) @util.Noparallel(blocking=False)
def download(self, check_size=False, blind_includes=False): def download(self, check_size=False, blind_includes=False):
@ -425,16 +427,7 @@ class Site(object):
self.updateWebsocket(updating=True) self.updateWebsocket(updating=True)
# Remove files that no longer in content.json # Remove files that no longer in content.json
for bad_file in self.bad_files.keys(): self.checkBadFiles()
file_info = self.content_manager.getFileInfo(bad_file)
if bad_file.endswith("content.json"):
if file_info is False:
del self.bad_files[bad_file]
self.log.debug("No info for file: %s, removing from bad_files" % bad_file)
else:
if file_info is False or not file_info.get("size"):
del self.bad_files[bad_file]
self.log.debug("No info for file: %s, removing from bad_files" % bad_file)
if announce: if announce:
self.announce() self.announce()