" % site.address
for key, peer in site.peers.items():
if peer.time_found:
time_found = int(time.time()-peer.time_found)/60
@@ -143,6 +143,7 @@ class UiRequestPlugin(object):
connection_id = peer.connection.id
else:
connection_id = None
+ yield "Optional files: %s " % len(peer.hashfield)
yield "(#%s, err: %s, found: %s min ago) %22s -
" % (connection_id, peer.connection_error, time_found, key)
yield "
|
"
yield ""
diff --git a/src/Config.py b/src/Config.py
index 8d7de4f8..d26dd899 100644
--- a/src/Config.py
+++ b/src/Config.py
@@ -8,7 +8,7 @@ class Config(object):
def __init__(self, argv):
self.version = "0.3.2"
- self.rev = 562
+ self.rev = 571
self.argv = argv
self.action = None
self.createParser()
diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py
index c9a68921..65e5d1c8 100644
--- a/src/Content/ContentManager.py
+++ b/src/Content/ContentManager.py
@@ -62,12 +62,38 @@ class ContentManager(object):
if old_hash != new_hash:
changed.append(content_inner_dir + relative_path)
+ # Check changed optional files
+ for relative_path, info in new_content.get("files_optional", {}).items():
+ file_inner_path = content_inner_dir + relative_path
+ new_hash = info["sha512"]
+ if old_content and old_content.get("files_optional", {}).get(relative_path): # We have the file in the old content
+ old_hash = old_content["files_optional"][relative_path].get("sha512")
+ if old_hash != new_hash and self.site.settings.get("autodownloadoptional"):
+ changed.append(content_inner_dir + relative_path) # Download new file
+ elif old_hash != new_hash and not self.site.settings.get("own"):
+ try:
+ self.site.storage.delete(file_inner_path)
+ self.log.debug("Deleted changed optional file: %s" % file_inner_path)
+ except Exception, err:
+ self.log.debug("Error deleting file %s: %s" % (file_inner_path, err))
+ else: # The file is not in the old content
+ if self.site.settings.get("autodownloadoptional"):
+ changed.append(content_inner_dir + relative_path) # Download new file
+
# Check deleted
if old_content:
- deleted = [
- content_inner_dir + key for key in old_content.get("files", {}) if key not in new_content.get("files", {})
- ]
- if deleted:
+ old_files = dict(
+ old_content.get("files", {}),
+ **old_content.get("files_optional", {})
+ )
+
+ new_files = dict(
+ new_content.get("files", {}),
+ **new_content.get("files_optional", {})
+ )
+
+ deleted = [content_inner_dir + key for key in old_files if key not in new_files]
+ if deleted and not self.site.settings.get("own"):
# Deleting files that no longer in content.json
for file_inner_path in deleted:
try:
diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py
index 418e7637..39f0f793 100644
--- a/src/File/FileRequest.py
+++ b/src/File/FileRequest.py
@@ -72,6 +72,8 @@ class FileRequest(object):
self.actionFindHashIds(params)
elif cmd == "setHashfield":
self.actionSetHashfield(params)
+ elif cmd == "siteReload":
+ self.actionSiteReload(params)
elif cmd == "ping":
self.actionPing()
else:
@@ -314,6 +316,17 @@ class FileRequest(object):
peer.hashfield.replaceFromString(params["hashfield_raw"])
self.response({"ok": "Updated"})
+ def actionSiteReload(self, params):
+ if self.connection.ip != "127.0.0.1" and self.connection.ip != config.ip_external:
+ self.response({"error": "Only local host allowed"})
+
+ site = self.sites.get(params["site"])
+ site.content_manager.loadContent(params["inner_path"], add_bad_files=False)
+ site.storage.verifyFiles(quick_check=True)
+ site.updateWebsocket()
+
+ self.response({"ok": "Reloaded"})
+
# Send a simple Pong! answer
def actionPing(self):
self.response("Pong!")
diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py
index 87c58a3b..a543d581 100644
--- a/src/Peer/Peer.py
+++ b/src/Peer/Peer.py
@@ -1,5 +1,6 @@
import logging
import time
+import sys
import gevent
@@ -60,7 +61,11 @@ class Peer(object):
self.connection = None
try:
- self.connection = self.site.connection_server.getConnection(self.ip, self.port)
+ if self.site:
+ self.connection = self.site.connection_server.getConnection(self.ip, self.port)
+ else:
+ self.connection = sys.modules["main"].file_server.getConnection(self.ip, self.port)
+
except Exception, err:
self.onConnectionError()
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %
diff --git a/src/Site/Site.py b/src/Site/Site.py
index 8938268d..4acd754a 100644
--- a/src/Site/Site.py
+++ b/src/Site/Site.py
@@ -130,6 +130,15 @@ class Site:
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
+ # Optionals files
+ if self.settings.get("autodownloadoptional"):
+ for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys():
+ file_inner_path = content_inner_dir + file_relative_path
+ # Start download and dont wait for finish, return the event
+ res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
+ if res is not True and res is not False: # Need downloading and file is allowed
+ file_threads.append(res) # Append evt
+
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
@@ -212,6 +221,7 @@ class Site:
# Wait for peers
if not self.peers:
+ self.announce()
for wait in range(10):
time.sleep(5+wait)
self.log.debug("Waiting for peers...")
@@ -258,10 +268,7 @@ class Site:
self.log.debug("Fallback to old-style update")
self.redownloadContents()
- if self.settings["own"]:
- self.storage.verifyFiles(quick_check=True) # Check files (need for optional files)
- else:
- self.storage.checkFiles(quick_check=True) # Quick check and mark bad files based on file size
+ self.storage.checkFiles(quick_check=True) # Quick check and mark bad files based on file size
changed, deleted = self.content_manager.loadContent("content.json")
diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py
index f9a4c3aa..bc276b3d 100644
--- a/src/Site/SiteStorage.py
+++ b/src/Site/SiteStorage.py
@@ -250,7 +250,7 @@ class SiteStorage:
return inner_path
# Verify all files sha512sum using content.json
- def verifyFiles(self, quick_check=False): # Fast = using file size
+ def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True):
bad_files = []
if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first
@@ -277,7 +277,8 @@ class SiteStorage:
if not ok:
self.log.debug("[CHANGED] %s" % file_inner_path)
- bad_files.append(file_inner_path)
+ if add_changed:
+ bad_files.append(file_inner_path)
# Optional files
optional_added = 0
@@ -288,6 +289,8 @@ class SiteStorage:
file_path = self.getPath(file_inner_path)
if not os.path.isfile(file_path):
self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"])
+ if add_optional:
+ bad_files.append(file_inner_path)
continue
if quick_check:
@@ -301,6 +304,8 @@ class SiteStorage:
else:
self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"])
optional_removed += 1
+ if add_optional:
+ bad_files.append(file_inner_path)
self.log.debug("[OPTIONAL CHANGED] %s" % file_inner_path)
self.log.debug(
@@ -313,10 +318,15 @@ class SiteStorage:
# Check and try to fix site files integrity
def checkFiles(self, quick_check=True):
s = time.time()
- bad_files = self.verifyFiles(quick_check)
+ bad_files = self.verifyFiles(
+ quick_check,
+ add_optional=self.site.settings.get("autodownloadoptional"),
+ add_changed=not self.site.settings.get("own") # Don't overwrite changed files if site owned
+ )
+ self.site.bad_files = {}
if bad_files:
for bad_file in bad_files:
- self.site.bad_files[bad_file] = self.site.bad_files.get("bad_file", 0) + 1
+ self.site.bad_files[bad_file] = 1
self.log.debug("Checked files in %.2fs... Quick:%s" % (time.time() - s, quick_check))
# Delete site's all file
diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py
index 4d762ccd..674de84d 100644
--- a/src/Ui/UiRequest.py
+++ b/src/Ui/UiRequest.py
@@ -138,6 +138,7 @@ class UiRequest(object):
headers = []
headers.append(("Version", "HTTP/1.1"))
headers.append(("Connection", "Keep-Alive"))
+ headers.append(("Keep-Alive", "max=25, timeout=30"))
headers.append(("Access-Control-Allow-Origin", "*")) # Allow json access
if self.env["REQUEST_METHOD"] == "OPTIONS":
# Allow json access
@@ -145,11 +146,11 @@ class UiRequest(object):
headers.append(("Access-Control-Allow-Credentials", "true"))
cacheable_type = (
- content_type == "text/css" or content_type.startswith("image") or
+ content_type == "text/css" or content_type.startswith("image") or content_type.startswith("video") or
self.env["REQUEST_METHOD"] == "OPTIONS" or content_type == "application/javascript"
)
- if status == 200 and cacheable_type: # Cache Css, Js, Image files for 10min
+ if status in (200, 206) and cacheable_type: # Cache Css, Js, Image files for 10min
headers.append(("Cache-Control", "public, max-age=600")) # Cache 10 min
else:
headers.append(("Cache-Control", "no-cache, no-store, private, must-revalidate, max-age=0")) # No caching at all
@@ -380,7 +381,7 @@ class UiRequest(object):
range_end = int(re.match(".*?-([0-9]+)", range).group(1))+1
else:
range_end = file_size
- extra_headers["Content-Length"] = range_end - range_start
+ extra_headers["Content-Length"] = str(range_end - range_start)
extra_headers["Content-Range"] = "bytes %s-%s/%s" % (range_start, range_end-1, file_size)
if range:
status = 206
diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py
index eb0a50b9..523d762b 100644
--- a/src/Ui/UiWebsocket.py
+++ b/src/Ui/UiWebsocket.py
@@ -152,6 +152,7 @@ class UiWebsocket(object):
if content: # Remove unnecessary data transfer
content = content.copy()
content["files"] = len(content.get("files", {}))
+ content["files_optional"] = len(content.get("files_optional", {}))
content["includes"] = len(content.get("includes", {}))
if "sign" in content:
del(content["sign"])
diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py
index ac7ce935..b969e07b 100644
--- a/src/Worker/WorkerManager.py
+++ b/src/Worker/WorkerManager.py
@@ -114,6 +114,18 @@ class WorkerManager:
continue # No peers found yet for the optional task
return task
+ def removeGoodFileTasks(self):
+ for task in self.tasks[:]:
+ if task["inner_path"] not in self.site.bad_files:
+ self.log.debug("No longer in bad_files, marking as good: %s" % task["inner_path"])
+ task["done"] = True
+ task["evt"].set(True)
+ self.tasks.remove(task)
+ if not self.tasks:
+ self.started_task_num = 0
+ self.site.updateWebsocket()
+
+
# New peers added to site
def onPeers(self):
self.startWorkers()
diff --git a/src/main.py b/src/main.py
index 045cb676..88be7682 100644
--- a/src/main.py
+++ b/src/main.py
@@ -226,18 +226,28 @@ class Actions(object):
global file_server
from Site import SiteManager
from File import FileServer # We need fileserver to handle incoming file requests
+ from Peer import Peer
logging.info("Creating FileServer....")
file_server = FileServer()
file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity
file_server.openport()
+
site = SiteManager.site_manager.list()[address]
site.settings["serving"] = True # Serving the site even if its disabled
+
+ # Notify local client on new content
+ if config.ip_external:
+ logging.info("Sending siteReload")
+ my_peer = Peer(config.ip_external, config.fileserver_port)
+ logging.info(my_peer.request("siteReload", {"site": site.address, "inner_path": inner_path}))
+
if peer_ip: # Announce ip specificed
site.addPeer(peer_ip, peer_port)
else: # Just ask the tracker
logging.info("Gathering peers from tracker")
site.announce() # Gather peers
+
published = site.publish(20, inner_path) # Push to 20 peers
if published > 0:
time.sleep(3)