Wait until downloadContent pool finishes
This commit is contained in:
parent
7c1da5da52
commit
3ccce46314
1 changed files with 1 additions and 0 deletions
|
@ -370,6 +370,7 @@ class Site(object):
|
||||||
del self.bad_files[aborted_inner_path]
|
del self.bad_files[aborted_inner_path]
|
||||||
self.worker_manager.removeSolvedFileTasks(mark_as_good=False)
|
self.worker_manager.removeSolvedFileTasks(mark_as_good=False)
|
||||||
break
|
break
|
||||||
|
pool.join()
|
||||||
self.log.debug("Ended downloadContent pool len: %s, skipped: %s" % (len(inner_paths), num_skipped))
|
self.log.debug("Ended downloadContent pool len: %s, skipped: %s" % (len(inner_paths), num_skipped))
|
||||||
|
|
||||||
def pooledDownloadFile(self, inner_paths, pool_size=100, only_if_bad=False):
|
def pooledDownloadFile(self, inner_paths, pool_size=100, only_if_bad=False):
|
||||||
|
|
Loading…
Reference in a new issue