From 6acc81cb50ce09f320f86b953dbaafa36c457a4a Mon Sep 17 00:00:00 2001 From: shortcutme Date: Mon, 7 Nov 2016 23:41:25 +0100 Subject: [PATCH] Retry bad files in separate thread --- src/Site/Site.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index 782589b1..a7a344e5 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -229,6 +229,7 @@ class Site(object): def retryBadFiles(self, force=False): self.log.debug("Retry %s bad files" % len(self.bad_files)) content_inner_paths = [] + file_inner_paths = [] for bad_file, tries in self.bad_files.items(): if force or random.randint(0, min(40, tries)) < 4: # Larger number tries = less likely to check every 15min if bad_file.endswith("content.json"): @@ -373,7 +374,7 @@ class Site(object): if self.bad_files: self.log.debug("Bad files: %s" % self.bad_files) - self.download() + gevent.spawn(self.retryBadFiles, force=True) if len(queried) == 0: # Failed to query modifications