oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign

Browse source
This commit is contained in:
HelloZeroNet 2015-11-09 00:44:03 +01:00
parent 2cf34c132f
commit 3587777ea8
17 changed files with 212 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/Config.py
View file

@ -8,7 +8,7 @@ class Config(object):
def __init__(self, argv):
self.version = "0.3.2"
self.rev = 562
self.rev = 571
self.argv = argv
self.action = None
self.createParser()

34
src/Content/ContentManager.py
View file

@ -62,12 +62,38 @@ class ContentManager(object):
if old_hash != new_hash:
changed.append(content_inner_dir + relative_path)
# Check changed optional files
for relative_path, info in new_content.get("files_optional", {}).items():
file_inner_path = content_inner_dir + relative_path
new_hash = info["sha512"]
if old_content and old_content.get("files_optional", {}).get(relative_path): # We have the file in the old content
old_hash = old_content["files_optional"][relative_path].get("sha512")
if old_hash != new_hash and self.site.settings.get("autodownloadoptional"):
changed.append(content_inner_dir + relative_path) # Download new file
elif old_hash != new_hash and not self.site.settings.get("own"):
try:
self.site.storage.delete(file_inner_path)
self.log.debug("Deleted changed optional file: %s" % file_inner_path)
except Exception, err:
self.log.debug("Error deleting file %s: %s" % (file_inner_path, err))
else: # The file is not in the old content
if self.site.settings.get("autodownloadoptional"):
changed.append(content_inner_dir + relative_path) # Download new file
# Check deleted
if old_content:
deleted = [
content_inner_dir + key for key in old_content.get("files", {}) if key not in new_content.get("files", {})
]
if deleted:
old_files = dict(
old_content.get("files", {}),
**old_content.get("files_optional", {})
)
new_files = dict(
new_content.get("files", {}),
**new_content.get("files_optional", {})
)
deleted = [content_inner_dir + key for key in old_files if key not in new_files]
if deleted and not self.site.settings.get("own"):
# Deleting files that no longer in content.json
for file_inner_path in deleted:
try:

13
src/File/FileRequest.py
View file

@ -72,6 +72,8 @@ class FileRequest(object):
self.actionFindHashIds(params)
elif cmd == "setHashfield":
self.actionSetHashfield(params)
elif cmd == "siteReload":
self.actionSiteReload(params)
elif cmd == "ping":
self.actionPing()
else:
@ -314,6 +316,17 @@ class FileRequest(object):
peer.hashfield.replaceFromString(params["hashfield_raw"])
self.response({"ok": "Updated"})
def actionSiteReload(self, params):
if self.connection.ip != "127.0.0.1" and self.connection.ip != config.ip_external:
self.response({"error": "Only local host allowed"})
site = self.sites.get(params["site"])
site.content_manager.loadContent(params["inner_path"], add_bad_files=False)
site.storage.verifyFiles(quick_check=True)
site.updateWebsocket()
self.response({"ok": "Reloaded"})
# Send a simple Pong! answer
def actionPing(self):
self.response("Pong!")

7
src/Peer/Peer.py
View file

@ -1,5 +1,6 @@
import logging
import time
import sys
import gevent
@ -60,7 +61,11 @@ class Peer(object):
self.connection = None
try:
self.connection = self.site.connection_server.getConnection(self.ip, self.port)
if self.site:
self.connection = self.site.connection_server.getConnection(self.ip, self.port)
else:
self.connection = sys.modules["main"].file_server.getConnection(self.ip, self.port)
except Exception, err:
self.onConnectionError()
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %

15
src/Site/Site.py
View file

@ -130,6 +130,15 @@ class Site:
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Optionals files
if self.settings.get("autodownloadoptional"):
for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Start download and dont wait for finish, return the event
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
@ -212,6 +221,7 @@ class Site:
# Wait for peers
if not self.peers:
self.announce()
for wait in range(10):
time.sleep(5+wait)
self.log.debug("Waiting for peers...")
@ -258,10 +268,7 @@ class Site:
self.log.debug("Fallback to old-style update")
self.redownloadContents()
if self.settings["own"]:
self.storage.verifyFiles(quick_check=True) # Check files (need for optional files)
else:
self.storage.checkFiles(quick_check=True) # Quick check and mark bad files based on file size
self.storage.checkFiles(quick_check=True) # Quick check and mark bad files based on file size
changed, deleted = self.content_manager.loadContent("content.json")

18
src/Site/SiteStorage.py
View file

@ -250,7 +250,7 @@ class SiteStorage:
return inner_path
# Verify all files sha512sum using content.json
def verifyFiles(self, quick_check=False): # Fast = using file size
def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True):
bad_files = []
if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first
@ -277,7 +277,8 @@ class SiteStorage:
if not ok:
self.log.debug("[CHANGED] %s" % file_inner_path)
bad_files.append(file_inner_path)
if add_changed:
bad_files.append(file_inner_path)
# Optional files
optional_added = 0
@ -288,6 +289,8 @@ class SiteStorage:
file_path = self.getPath(file_inner_path)
if not os.path.isfile(file_path):
self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"])
if add_optional:
bad_files.append(file_inner_path)
continue
if quick_check:
@ -301,6 +304,8 @@ class SiteStorage:
else:
self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"])
optional_removed += 1
if add_optional:
bad_files.append(file_inner_path)
self.log.debug("[OPTIONAL CHANGED] %s" % file_inner_path)
self.log.debug(
@ -313,10 +318,15 @@ class SiteStorage:
# Check and try to fix site files integrity
def checkFiles(self, quick_check=True):
s = time.time()
bad_files = self.verifyFiles(quick_check)
bad_files = self.verifyFiles(
quick_check,
add_optional=self.site.settings.get("autodownloadoptional"),
add_changed=not self.site.settings.get("own") # Don't overwrite changed files if site owned
)
self.site.bad_files = {}
if bad_files:
for bad_file in bad_files:
self.site.bad_files[bad_file] = self.site.bad_files.get("bad_file", 0) + 1
self.site.bad_files[bad_file] = 1
self.log.debug("Checked files in %.2fs... Quick:%s" % (time.time() - s, quick_check))
# Delete site's all file

7
src/Ui/UiRequest.py
View file

@ -138,6 +138,7 @@ class UiRequest(object):
headers = []
headers.append(("Version", "HTTP/1.1"))
headers.append(("Connection", "Keep-Alive"))
headers.append(("Keep-Alive", "max=25, timeout=30"))
headers.append(("Access-Control-Allow-Origin", "*")) # Allow json access
if self.env["REQUEST_METHOD"] == "OPTIONS":
# Allow json access
@ -145,11 +146,11 @@ class UiRequest(object):
headers.append(("Access-Control-Allow-Credentials", "true"))
cacheable_type = (
content_type == "text/css" or content_type.startswith("image") or
content_type == "text/css" or content_type.startswith("image") or content_type.startswith("video") or
self.env["REQUEST_METHOD"] == "OPTIONS" or content_type == "application/javascript"
)
if status == 200 and cacheable_type: # Cache Css, Js, Image files for 10min
if status in (200, 206) and cacheable_type: # Cache Css, Js, Image files for 10min
headers.append(("Cache-Control", "public, max-age=600")) # Cache 10 min
else:
headers.append(("Cache-Control", "no-cache, no-store, private, must-revalidate, max-age=0")) # No caching at all
@ -380,7 +381,7 @@ class UiRequest(object):
range_end = int(re.match(".*?-([0-9]+)", range).group(1))+1
else:
range_end = file_size
extra_headers["Content-Length"] = range_end - range_start
extra_headers["Content-Length"] = str(range_end - range_start)
extra_headers["Content-Range"] = "bytes %s-%s/%s" % (range_start, range_end-1, file_size)
if range:
status = 206

1
src/Ui/UiWebsocket.py
View file

@ -152,6 +152,7 @@ class UiWebsocket(object):
if content: # Remove unnecessary data transfer
content = content.copy()
content["files"] = len(content.get("files", {}))
content["files_optional"] = len(content.get("files_optional", {}))
content["includes"] = len(content.get("includes", {}))
if "sign" in content:
del(content["sign"])

12
src/Worker/WorkerManager.py
View file

@ -114,6 +114,18 @@ class WorkerManager:
continue # No peers found yet for the optional task
return task
def removeGoodFileTasks(self):
for task in self.tasks[:]:
if task["inner_path"] not in self.site.bad_files:
self.log.debug("No longer in bad_files, marking as good: %s" % task["inner_path"])
task["done"] = True
task["evt"].set(True)
self.tasks.remove(task)
if not self.tasks:
self.started_task_num = 0
self.site.updateWebsocket()
# New peers added to site
def onPeers(self):
self.startWorkers()

10
src/main.py
View file

@ -226,18 +226,28 @@ class Actions(object):
global file_server
from Site import SiteManager
from File import FileServer # We need fileserver to handle incoming file requests
from Peer import Peer
logging.info("Creating FileServer....")
file_server = FileServer()
file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity
file_server.openport()
site = SiteManager.site_manager.list()[address]
site.settings["serving"] = True # Serving the site even if its disabled
# Notify local client on new content
if config.ip_external:
logging.info("Sending siteReload")
my_peer = Peer(config.ip_external, config.fileserver_port)
logging.info(my_peer.request("siteReload", {"site": site.address, "inner_path": inner_path}))
if peer_ip: # Announce ip specificed
site.addPeer(peer_ip, peer_port)
else: # Just ask the tracker
logging.info("Gathering peers from tracker")
site.announce() # Gather peers
published = site.publish(20, inner_path) # Push to 20 peers
if published > 0:
time.sleep(3)