oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support streaming bigfile for zip file listing and reading

Browse source
This commit is contained in:
shortcutme 2018-08-16 16:21:21 +02:00
parent 5854beebc6
commit cfce057783
No known key found for this signature in database
GPG key ID: 5B63BAE6CB9613AE
1 changed files with 36 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
plugins/FilePack/FilePackPlugin.py
View file

@ -5,6 +5,7 @@ import gevent
from Plugin import PluginManager from Plugin import PluginManager
from Config import config from Config import config
from Debug import Debug
# Keep archive open for faster reponse times for large sites # Keep archive open for faster reponse times for large sites
@ -16,24 +17,25 @@ def closeArchive(archive_path):
del archive_cache[archive_path] del archive_cache[archive_path]
def openArchive(archive_path): def openArchive(archive_path, file_obj=None):
if archive_path not in archive_cache: if archive_path not in archive_cache:
if archive_path.endswith("tar.gz"): if archive_path.endswith("tar.gz"):
import tarfile import tarfile
archive_cache[archive_path] = tarfile.open(archive_path, "r:gz") archive_cache[archive_path] = tarfile.open(file_obj or archive_path, "r:gz")
elif archive_path.endswith("tar.bz2"): elif archive_path.endswith("tar.bz2"):
import tarfile import tarfile
archive_cache[archive_path] = tarfile.open(archive_path, "r:bz2") archive_cache[archive_path] = tarfile.open(file_obj or archive_path, "r:bz2")
else: else:
import zipfile import zipfile
archive_cache[archive_path] = zipfile.ZipFile(archive_path) archive_cache[archive_path] = zipfile.ZipFile(file_obj or archive_path)
gevent.spawn_later(5, lambda: closeArchive(archive_path)) # Close after 5 sec gevent.spawn_later(5, lambda: closeArchive(archive_path)) # Close after 5 sec
archive = archive_cache[archive_path] archive = archive_cache[archive_path]
return archive return archive
def openArchiveFile(archive_path, path_within):
archive = openArchive(archive_path) def openArchiveFile(archive_path, path_within, file_obj=None):
archive = openArchive(archive_path, file_obj=file_obj)
if archive_path.endswith(".zip"): if archive_path.endswith(".zip"):
return archive.open(path_within) return archive.open(path_within)
else: else:
@ -44,20 +46,24 @@ def openArchiveFile(archive_path, path_within):
class UiRequestPlugin(object): class UiRequestPlugin(object):
def actionSiteMedia(self, path, **kwargs): def actionSiteMedia(self, path, **kwargs):
if ".zip/" in path or ".tar.gz/" in path: if ".zip/" in path or ".tar.gz/" in path:
file_obj = None
path_parts = self.parsePath(path) path_parts = self.parsePath(path)
file_path = u"%s/%s/%s" % (config.data_dir, path_parts["address"], path_parts["inner_path"].decode("utf8")) file_path = u"%s/%s/%s" % (config.data_dir, path_parts["address"], path_parts["inner_path"].decode("utf8"))
match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", file_path) match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", file_path)
archive_path, path_within = match.groups() archive_path, path_within = match.groups()
if not os.path.isfile(archive_path): if archive_path not in archive_cache:
site = self.server.site_manager.get(path_parts["address"]) site = self.server.site_manager.get(path_parts["address"])
if not site: if not site:
return self.actionSiteAddPrompt(path) return self.actionSiteAddPrompt(path)
# Wait until file downloads archive_inner_path = site.storage.getInnerPath(archive_path)
result = site.needFile(site.storage.getInnerPath(archive_path), priority=10) if not os.path.isfile(archive_path):
# Send virutal file path download finished event to remove loading screen # Wait until file downloads
site.updateWebsocket(file_done=site.storage.getInnerPath(file_path)) result = site.needFile(archive_inner_path, priority=10)
if not result: # Send virutal file path download finished event to remove loading screen
return self.error404(path) site.updateWebsocket(file_done=archive_inner_path)
if not result:
return self.error404(archive_inner_path)
file_obj = site.storage.openBigfile(archive_inner_path)
header_allow_ajax = False header_allow_ajax = False
if self.get.get("ajax_key"): if self.get.get("ajax_key"):
@ -68,12 +74,12 @@ class UiRequestPlugin(object):
return self.error403("Invalid ajax_key") return self.error403("Invalid ajax_key")
try: try:
file = openArchiveFile(archive_path, path_within) file = openArchiveFile(archive_path, path_within, file_obj=file_obj)
content_type = self.getContentType(file_path) content_type = self.getContentType(file_path)
self.sendHeader(200, content_type=content_type, noscript=kwargs.get("header_noscript", False), allow_ajax=header_allow_ajax) self.sendHeader(200, content_type=content_type, noscript=kwargs.get("header_noscript", False), allow_ajax=header_allow_ajax)
return self.streamFile(file) return self.streamFile(file)
except Exception as err: except Exception as err:
self.log.debug("Error opening archive file: %s" % err) self.log.debug("Error opening archive file: %s" % Debug.formatException(err))
return self.error404(path) return self.error404(path)
return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs) return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs)
@ -105,7 +111,21 @@ class SiteStoragePlugin(object):
if ".zip" in inner_path or ".tar.gz" in inner_path: if ".zip" in inner_path or ".tar.gz" in inner_path:
match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path) match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path)
archive_inner_path, path_within = match.groups() archive_inner_path, path_within = match.groups()
archive = openArchive(self.getPath(archive_inner_path)) archive_path = self.getPath(archive_inner_path)
file_obj = None
if archive_path not in archive_cache:
if not os.path.isfile(archive_path):
result = self.site.needFile(archive_inner_path, priority=10)
self.site.updateWebsocket(file_done=archive_inner_path)
if not result:
raise Exception("Unable to download file")
file_obj = self.site.storage.openBigfile(archive_inner_path)
try:
archive = openArchive(archive_path, file_obj=file_obj)
except Exception as err:
raise Exception("Unable to download file: %s" % err)
if archive_inner_path.endswith(".zip"): if archive_inner_path.endswith(".zip"):
namelist = [name for name in archive.namelist() if not name.endswith("/")] namelist = [name for name in archive.namelist() if not name.endswith("/")]
else: else: