oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Better logging in site download content

Browse source
This commit is contained in:
shortcutme 2019-12-21 03:02:36 +01:00
parent 975f53b95b
commit e16ace433c
No known key found for this signature in database
GPG key ID: 5B63BAE6CB9613AE
1 changed files with 18 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
src/Site/Site.py
View file

@ -158,7 +158,10 @@ class Site(object):
def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}): def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}):
s = time.time() s = time.time()
if config.verbose: if config.verbose:
self.log.debug("Downloading %s..." % inner_path) self.log.debug(
"DownloadContent %s: Started. (download_files: %s, check_modifications: %s, diffs: %s)..." %
(inner_path, download_files, check_modifications, diffs.keys())
)
if not inner_path.endswith("content.json"): if not inner_path.endswith("content.json"):
return False return False
@ -166,15 +169,20 @@ class Site(object):
found = self.needFile(inner_path, update=self.bad_files.get(inner_path)) found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
content_inner_dir = helper.getDirname(inner_path) content_inner_dir = helper.getDirname(inner_path)
if not found: if not found:
self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications)) self.log.debug("DownloadContent %s: Download failed, check_modifications: %s" % (inner_path, check_modifications))
if check_modifications: # Download failed, but check modifications if its succed later if check_modifications: # Download failed, but check modifications if its succed later
self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications") self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications")
return False # Could not download content.json return False # Could not download content.json
if config.verbose: if config.verbose:
self.log.debug("Got %s" % inner_path) self.log.debug("DownloadContent got %s" % inner_path)
sub_s = time.time()
changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False) changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False)
if config.verbose:
self.log.debug("DownloadContent %s: loadContent done in %.3fs" % (inner_path, time.time() - sub_s))
if inner_path == "content.json": if inner_path == "content.json":
self.saveSettings() self.saveSettings()
@ -187,7 +195,7 @@ class Site(object):
content_size = len(json.dumps(self.content_manager.contents[inner_path], indent=1)) + sum([file["size"] for file in list(self.content_manager.contents[inner_path].get("files", {}).values()) if file["size"] >= 0]) # Size of new content content_size = len(json.dumps(self.content_manager.contents[inner_path], indent=1)) + sum([file["size"] for file in list(self.content_manager.contents[inner_path].get("files", {}).values()) if file["size"] >= 0]) # Size of new content
if site_size_limit < content_size: if site_size_limit < content_size:
# Not enought don't download anything # Not enought don't download anything
self.log.debug("Size limit reached (site too big please increase limit): %.2f MB > %.2f MB" % (content_size / 1024 / 1024, site_size_limit / 1024 / 1024)) self.log.debug("DownloadContent Size limit reached (site too big please increase limit): %.2f MB > %.2f MB" % (content_size / 1024 / 1024, site_size_limit / 1024 / 1024))
return False return False
# Start download files # Start download files
@ -221,11 +229,11 @@ class Site(object):
time_on_done = time.time() - s time_on_done = time.time() - s
self.log.debug( self.log.debug(
"Patched successfully: %s (diff: %.3fs, verify: %.3fs, write: %.3fs, on_done: %.3fs)" % "DownloadContent Patched successfully: %s (diff: %.3fs, verify: %.3fs, write: %.3fs, on_done: %.3fs)" %
(file_inner_path, time_diff, time_verify, time_write, time_on_done) (file_inner_path, time_diff, time_verify, time_write, time_on_done)
) )
except Exception as err: except Exception as err:
self.log.debug("Failed to patch %s: %s" % (file_inner_path, err)) self.log.debug("DownloadContent Failed to patch %s: %s" % (file_inner_path, err))
diff_success = False diff_success = False
if not diff_success: if not diff_success:
@ -259,19 +267,19 @@ class Site(object):
include_threads.append(include_thread) include_threads.append(include_thread)
if config.verbose: if config.verbose:
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads))) self.log.debug("DownloadContent %s: Downloading %s includes..." % (inner_path, len(include_threads)))
gevent.joinall(include_threads) gevent.joinall(include_threads)
if config.verbose: if config.verbose:
self.log.debug("%s: Includes download ended" % inner_path) self.log.debug("DownloadContent %s: Includes download ended" % inner_path)
if check_modifications: # Check if every file is up-to-date if check_modifications: # Check if every file is up-to-date
self.checkModifications(0) self.checkModifications(0)
if config.verbose: if config.verbose:
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed))) self.log.debug("DownloadContent %s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads) gevent.joinall(file_threads)
if config.verbose: if config.verbose:
self.log.debug("%s: DownloadContent ended in %.3fs (tasks left: %s)" % ( self.log.debug("DownloadContent %s: ended in %.3fs (tasks left: %s)" % (
inner_path, time.time() - s, len(self.worker_manager.tasks) inner_path, time.time() - s, len(self.worker_manager.tasks)
)) ))