Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign

This commit is contained in:
HelloZeroNet 2015-11-09 00:44:03 +01:00
parent 2cf34c132f
commit 3587777ea8
17 changed files with 212 additions and 41 deletions

View file

@ -201,6 +201,55 @@ class UiWebsocketPlugin(object):
</li> </li>
""".format(**locals())) """.format(**locals()))
def sidebarRenderOptionalFileStats(self, body, site):
size_total = 0.0
size_downloaded = 0.0
for content in site.content_manager.contents.values():
if "files_optional" not in content:
continue
for file_name, file_details in content["files_optional"].items():
size_total += file_details["size"]
if site.content_manager.hashfield.hasHash(file_details["sha512"]):
size_downloaded += file_details["size"]
if not size_total:
return False
percent_downloaded = size_downloaded / size_total
size_formatted_total = size_total / 1024 / 1024
size_formatted_downloaded = size_downloaded / 1024 / 1024
body.append("""
<li>
<label>Optional files</label>
<ul class='graph'>
<li style='width: 100%' class='total back-black' title="Total size"></li>
<li style='width: {percent_downloaded:.0%}' class='connected back-green' title='Downloaded files'></li>
</ul>
<ul class='graph-legend'>
<li class='color-green'><span>Downloaded:</span><b>{size_formatted_downloaded:.2f}MB</b></li>
<li class='color-black'><span>Total:</span><b>{size_formatted_total:.2f}MB</b></li>
</ul>
</li>
""".format(**locals()))
return True
def sidebarRenderOptionalFileSettings(self, body, site):
if self.site.settings.get("autodownloadoptional"):
checked = "checked='checked'"
else:
checked = ""
body.append("""
<li>
<label>Download and help distribute all files</label>
<input type="checkbox" class="checkbox" id="checkbox-autodownloadoptional" {checked}/><div class="checkbox-skin"></div>
</li>
""".format(**locals()))
def sidebarRenderDbOptions(self, body, site): def sidebarRenderDbOptions(self, body, site):
if not site.storage.db: if not site.storage.db:
return False return False
@ -232,7 +281,7 @@ class UiWebsocketPlugin(object):
checked = "" checked = ""
body.append(""" body.append("""
<h2 class='owned-title'>Owned site settings</h2> <h2 class='owned-title'>This is my site</h2>
<input type="checkbox" class="checkbox" id="checkbox-owned" {checked}/><div class="checkbox-skin"></div> <input type="checkbox" class="checkbox" id="checkbox-owned" {checked}/><div class="checkbox-skin"></div>
""".format(**locals())) """.format(**locals()))
@ -296,6 +345,9 @@ class UiWebsocketPlugin(object):
self.sidebarRenderTransferStats(body, site) self.sidebarRenderTransferStats(body, site)
self.sidebarRenderFileStats(body, site) self.sidebarRenderFileStats(body, site)
self.sidebarRenderSizeLimit(body, site) self.sidebarRenderSizeLimit(body, site)
has_optional = self.sidebarRenderOptionalFileStats(body, site)
if has_optional:
self.sidebarRenderOptionalFileSettings(body, site)
self.sidebarRenderDbOptions(body, site) self.sidebarRenderDbOptions(body, site)
self.sidebarRenderIdentity(body, site) self.sidebarRenderIdentity(body, site)
@ -405,3 +457,12 @@ class UiWebsocketPlugin(object):
if "ADMIN" not in permissions: if "ADMIN" not in permissions:
return self.response(to, "You don't have permission to run this command") return self.response(to, "You don't have permission to run this command")
self.site.settings["own"] = bool(owned) self.site.settings["own"] = bool(owned)
def actionSiteSetAutodownloadoptional(self, to, owned):
permissions = self.getPermissions(to)
if "ADMIN" not in permissions:
return self.response(to, "You don't have permission to run this command")
self.site.settings["autodownloadoptional"] = bool(owned)
self.site.update()
self.site.worker_manager.removeGoodFileTasks()

View file

@ -15,9 +15,9 @@ window.initScrollable = function () {
// *Calculation of how tall scroller should be // *Calculation of how tall scroller should be
var visibleRatio = scrollContainer.offsetHeight / scrollContentWrapper.scrollHeight; var visibleRatio = scrollContainer.offsetHeight / scrollContentWrapper.scrollHeight;
if (visibleRatio == 1) if (visibleRatio == 1)
scroller.style.display = "none" scroller.style.display = "none";
else else
scroller.style.display = "block" scroller.style.display = "block";
return visibleRatio * scrollContainer.offsetHeight; return visibleRatio * scrollContainer.offsetHeight;
} }
@ -32,13 +32,13 @@ window.initScrollable = function () {
normalizedPosition = evt.pageY; normalizedPosition = evt.pageY;
contentPosition = scrollContentWrapper.scrollTop; contentPosition = scrollContentWrapper.scrollTop;
scrollerBeingDragged = true; scrollerBeingDragged = true;
window.addEventListener('mousemove', scrollBarScroll) window.addEventListener('mousemove', scrollBarScroll);
return false return false;
} }
function stopDrag(evt) { function stopDrag(evt) {
scrollerBeingDragged = false; scrollerBeingDragged = false;
window.removeEventListener('mousemove', scrollBarScroll) window.removeEventListener('mousemove', scrollBarScroll);
} }
function scrollBarScroll(evt) { function scrollBarScroll(evt) {
@ -51,7 +51,7 @@ window.initScrollable = function () {
} }
function updateHeight() { function updateHeight() {
scrollerHeight = calculateScrollerHeight()-10; scrollerHeight = calculateScrollerHeight() - 10;
scroller.style.height = scrollerHeight + 'px'; scroller.style.height = scrollerHeight + 'px';
} }
@ -62,9 +62,9 @@ window.initScrollable = function () {
scroller.className = 'scroller'; scroller.className = 'scroller';
// determine how big scroller should be based on content // determine how big scroller should be based on content
scrollerHeight = calculateScrollerHeight()-10; scrollerHeight = calculateScrollerHeight() - 10;
if (scrollerHeight / scrollContainer.offsetHeight < 1){ if (scrollerHeight / scrollContainer.offsetHeight < 1) {
// *If there is a need to have scroll bar based on content size // *If there is a need to have scroll bar based on content size
scroller.style.height = scrollerHeight + 'px'; scroller.style.height = scrollerHeight + 'px';
@ -87,5 +87,5 @@ window.initScrollable = function () {
// *** Listeners *** // *** Listeners ***
scrollContentWrapper.addEventListener('scroll', moveScroller); scrollContentWrapper.addEventListener('scroll', moveScroller);
return updateHeight return updateHeight;
}; };

View file

@ -220,6 +220,14 @@ class Sidebar extends Class
@updateHtmlTag() @updateHtmlTag()
return false return false
# Owned checkbox
@tag.find("#checkbox-owned").on "click", =>
wrapper.ws.cmd "siteSetOwned", [@tag.find("#checkbox-owned").is(":checked")]
# Owned checkbox
@tag.find("#checkbox-autodownloadoptional").on "click", =>
wrapper.ws.cmd "siteSetAutodownloadoptional", [@tag.find("#checkbox-autodownloadoptional").is(":checked")]
# Change identity button # Change identity button
@tag.find("#button-identity").on "click", => @tag.find("#button-identity").on "click", =>
wrapper.ws.cmd "certSelect" wrapper.ws.cmd "certSelect"

View file

@ -21,7 +21,10 @@
.sidebar .fields { padding: 0px; list-style-type: none; width: 355px; } .sidebar .fields { padding: 0px; list-style-type: none; width: 355px; }
.sidebar .fields > li, .sidebar .fields .settings-owned > li { margin-bottom: 30px } .sidebar .fields > li, .sidebar .fields .settings-owned > li { margin-bottom: 30px }
.sidebar .fields > li:after, .sidebar .fields .settings-owned > li:after { clear: both; content: ''; display: block } .sidebar .fields > li:after, .sidebar .fields .settings-owned > li:after { clear: both; content: ''; display: block }
.sidebar .fields label { font-family: Consolas, monospace; text-transform: uppercase; font-size: 13px; color: #ACACAC; display: block; margin-bottom: 10px; } .sidebar .fields label {
font-family: Consolas, monospace; text-transform: uppercase; font-size: 13px; color: #ACACAC; display: inline-block; margin-bottom: 10px;
vertical-align: text-bottom; margin-right: 10px;
}
.sidebar .fields label small { font-weight: normal; color: white; text-transform: none; } .sidebar .fields label small { font-weight: normal; color: white; text-transform: none; }
.sidebar .fields .text { background-color: black; border: 0px; padding: 10px; color: white; border-radius: 3px; width: 250px; font-family: Consolas, monospace; } .sidebar .fields .text { background-color: black; border: 0px; padding: 10px; color: white; border-radius: 3px; width: 250px; font-family: Consolas, monospace; }
.sidebar .fields .text.long { width: 330px; font-size: 72%; } .sidebar .fields .text.long { width: 330px; font-size: 72%; }
@ -52,7 +55,7 @@
/* GRAPH */ /* GRAPH */
.graph { padding: 0px; list-style-type: none; width: 351px; background-color: black; height: 10px; border-radius: 8px; overflow: hidden; position: relative;} .graph { padding: 0px; list-style-type: none; width: 351px; background-color: black; height: 10px; border-radius: 8px; overflow: hidden; position: relative;}
.graph li { height: 100%; position: absolute; } .graph li { height: 100%; position: absolute; transition: all 0.3s; }
.graph-stacked li { position: static; float: left; } .graph-stacked li { position: static; float: left; }
.graph-legend { padding: 0px; list-style-type: none; margin-top: 13px; font-family: Consolas, "Andale Mono", monospace; font-size: 13px; text-transform: capitalize; } .graph-legend { padding: 0px; list-style-type: none; margin-top: 13px; font-family: Consolas, "Andale Mono", monospace; font-size: 13px; text-transform: capitalize; }

View file

@ -75,7 +75,10 @@
.sidebar .fields { padding: 0px; list-style-type: none; width: 355px; } .sidebar .fields { padding: 0px; list-style-type: none; width: 355px; }
.sidebar .fields > li, .sidebar .fields .settings-owned > li { margin-bottom: 30px } .sidebar .fields > li, .sidebar .fields .settings-owned > li { margin-bottom: 30px }
.sidebar .fields > li:after, .sidebar .fields .settings-owned > li:after { clear: both; content: ''; display: block } .sidebar .fields > li:after, .sidebar .fields .settings-owned > li:after { clear: both; content: ''; display: block }
.sidebar .fields label { font-family: Consolas, monospace; text-transform: uppercase; font-size: 13px; color: #ACACAC; display: block; margin-bottom: 10px; } .sidebar .fields label {
font-family: Consolas, monospace; text-transform: uppercase; font-size: 13px; color: #ACACAC; display: inline-block; margin-bottom: 10px;
vertical-align: text-bottom; margin-right: 10px;
}
.sidebar .fields label small { font-weight: normal; color: white; text-transform: none; } .sidebar .fields label small { font-weight: normal; color: white; text-transform: none; }
.sidebar .fields .text { background-color: black; border: 0px; padding: 10px; color: white; -webkit-border-radius: 3px; -moz-border-radius: 3px; -o-border-radius: 3px; -ms-border-radius: 3px; border-radius: 3px ; width: 250px; font-family: Consolas, monospace; } .sidebar .fields .text { background-color: black; border: 0px; padding: 10px; color: white; -webkit-border-radius: 3px; -moz-border-radius: 3px; -o-border-radius: 3px; -ms-border-radius: 3px; border-radius: 3px ; width: 250px; font-family: Consolas, monospace; }
.sidebar .fields .text.long { width: 330px; font-size: 72%; } .sidebar .fields .text.long { width: 330px; font-size: 72%; }
@ -106,7 +109,7 @@
/* GRAPH */ /* GRAPH */
.graph { padding: 0px; list-style-type: none; width: 351px; background-color: black; height: 10px; -webkit-border-radius: 8px; -moz-border-radius: 8px; -o-border-radius: 8px; -ms-border-radius: 8px; border-radius: 8px ; overflow: hidden; position: relative;} .graph { padding: 0px; list-style-type: none; width: 351px; background-color: black; height: 10px; -webkit-border-radius: 8px; -moz-border-radius: 8px; -o-border-radius: 8px; -ms-border-radius: 8px; border-radius: 8px ; overflow: hidden; position: relative;}
.graph li { height: 100%; position: absolute; } .graph li { height: 100%; position: absolute; -webkit-transition: all 0.3s; -moz-transition: all 0.3s; -o-transition: all 0.3s; -ms-transition: all 0.3s; transition: all 0.3s ; }
.graph-stacked li { position: static; float: left; } .graph-stacked li { position: static; float: left; }
.graph-legend { padding: 0px; list-style-type: none; margin-top: 13px; font-family: Consolas, "Andale Mono", monospace; font-size: 13px; text-transform: capitalize; } .graph-legend { padding: 0px; list-style-type: none; margin-top: 13px; font-family: Consolas, "Andale Mono", monospace; font-size: 13px; text-transform: capitalize; }

View file

@ -77,9 +77,9 @@ window.initScrollable = function () {
// *Calculation of how tall scroller should be // *Calculation of how tall scroller should be
var visibleRatio = scrollContainer.offsetHeight / scrollContentWrapper.scrollHeight; var visibleRatio = scrollContainer.offsetHeight / scrollContentWrapper.scrollHeight;
if (visibleRatio == 1) if (visibleRatio == 1)
scroller.style.display = "none" scroller.style.display = "none";
else else
scroller.style.display = "block" scroller.style.display = "block";
return visibleRatio * scrollContainer.offsetHeight; return visibleRatio * scrollContainer.offsetHeight;
} }
@ -94,13 +94,13 @@ window.initScrollable = function () {
normalizedPosition = evt.pageY; normalizedPosition = evt.pageY;
contentPosition = scrollContentWrapper.scrollTop; contentPosition = scrollContentWrapper.scrollTop;
scrollerBeingDragged = true; scrollerBeingDragged = true;
window.addEventListener('mousemove', scrollBarScroll) window.addEventListener('mousemove', scrollBarScroll);
return false return false;
} }
function stopDrag(evt) { function stopDrag(evt) {
scrollerBeingDragged = false; scrollerBeingDragged = false;
window.removeEventListener('mousemove', scrollBarScroll) window.removeEventListener('mousemove', scrollBarScroll);
} }
function scrollBarScroll(evt) { function scrollBarScroll(evt) {
@ -113,7 +113,7 @@ window.initScrollable = function () {
} }
function updateHeight() { function updateHeight() {
scrollerHeight = calculateScrollerHeight()-10; scrollerHeight = calculateScrollerHeight() - 10;
scroller.style.height = scrollerHeight + 'px'; scroller.style.height = scrollerHeight + 'px';
} }
@ -124,9 +124,9 @@ window.initScrollable = function () {
scroller.className = 'scroller'; scroller.className = 'scroller';
// determine how big scroller should be based on content // determine how big scroller should be based on content
scrollerHeight = calculateScrollerHeight()-10; scrollerHeight = calculateScrollerHeight() - 10;
if (scrollerHeight / scrollContainer.offsetHeight < 1){ if (scrollerHeight / scrollContainer.offsetHeight < 1) {
// *If there is a need to have scroll bar based on content size // *If there is a need to have scroll bar based on content size
scroller.style.height = scrollerHeight + 'px'; scroller.style.height = scrollerHeight + 'px';
@ -149,7 +149,7 @@ window.initScrollable = function () {
// *** Listeners *** // *** Listeners ***
scrollContentWrapper.addEventListener('scroll', moveScroller); scrollContentWrapper.addEventListener('scroll', moveScroller);
return updateHeight return updateHeight;
}; };
@ -398,6 +398,16 @@ window.initScrollable = function () {
return false; return false;
}; };
})(this)); })(this));
this.tag.find("#checkbox-owned").on("click", (function(_this) {
return function() {
return wrapper.ws.cmd("siteSetOwned", [_this.tag.find("#checkbox-owned").is(":checked")]);
};
})(this));
this.tag.find("#checkbox-autodownloadoptional").on("click", (function(_this) {
return function() {
return wrapper.ws.cmd("siteSetAutodownloadoptional", [_this.tag.find("#checkbox-autodownloadoptional").is(":checked")]);
};
})(this));
this.tag.find("#button-identity").on("click", (function(_this) { this.tag.find("#button-identity").on("click", (function(_this) {
return function() { return function() {
wrapper.ws.cmd("certSelect"); wrapper.ws.cmd("certSelect");

View file

@ -133,7 +133,7 @@ class UiRequestPlugin(object):
("%.0fkB", site.settings.get("bytes_sent", 0) / 1024), ("%.0fkB", site.settings.get("bytes_sent", 0) / 1024),
("%.0fkB", site.settings.get("bytes_recv", 0) / 1024), ("%.0fkB", site.settings.get("bytes_recv", 0) / 1024),
]) ])
yield "<tr><td id='peers_%s' style='display: none; white-space: pre'>" % site.address yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=2>" % site.address
for key, peer in site.peers.items(): for key, peer in site.peers.items():
if peer.time_found: if peer.time_found:
time_found = int(time.time()-peer.time_found)/60 time_found = int(time.time()-peer.time_found)/60
@ -143,6 +143,7 @@ class UiRequestPlugin(object):
connection_id = peer.connection.id connection_id = peer.connection.id
else: else:
connection_id = None connection_id = None
yield "Optional files: %s " % len(peer.hashfield)
yield "(#%s, err: %s, found: %s min ago) %22s -<br>" % (connection_id, peer.connection_error, time_found, key) yield "(#%s, err: %s, found: %s min ago) %22s -<br>" % (connection_id, peer.connection_error, time_found, key)
yield "<br></td></tr>" yield "<br></td></tr>"
yield "</table>" yield "</table>"

View file

@ -8,7 +8,7 @@ class Config(object):
def __init__(self, argv): def __init__(self, argv):
self.version = "0.3.2" self.version = "0.3.2"
self.rev = 562 self.rev = 571
self.argv = argv self.argv = argv
self.action = None self.action = None
self.createParser() self.createParser()

View file

@ -62,12 +62,38 @@ class ContentManager(object):
if old_hash != new_hash: if old_hash != new_hash:
changed.append(content_inner_dir + relative_path) changed.append(content_inner_dir + relative_path)
# Check changed optional files
for relative_path, info in new_content.get("files_optional", {}).items():
file_inner_path = content_inner_dir + relative_path
new_hash = info["sha512"]
if old_content and old_content.get("files_optional", {}).get(relative_path): # We have the file in the old content
old_hash = old_content["files_optional"][relative_path].get("sha512")
if old_hash != new_hash and self.site.settings.get("autodownloadoptional"):
changed.append(content_inner_dir + relative_path) # Download new file
elif old_hash != new_hash and not self.site.settings.get("own"):
try:
self.site.storage.delete(file_inner_path)
self.log.debug("Deleted changed optional file: %s" % file_inner_path)
except Exception, err:
self.log.debug("Error deleting file %s: %s" % (file_inner_path, err))
else: # The file is not in the old content
if self.site.settings.get("autodownloadoptional"):
changed.append(content_inner_dir + relative_path) # Download new file
# Check deleted # Check deleted
if old_content: if old_content:
deleted = [ old_files = dict(
content_inner_dir + key for key in old_content.get("files", {}) if key not in new_content.get("files", {}) old_content.get("files", {}),
] **old_content.get("files_optional", {})
if deleted: )
new_files = dict(
new_content.get("files", {}),
**new_content.get("files_optional", {})
)
deleted = [content_inner_dir + key for key in old_files if key not in new_files]
if deleted and not self.site.settings.get("own"):
# Deleting files that no longer in content.json # Deleting files that no longer in content.json
for file_inner_path in deleted: for file_inner_path in deleted:
try: try:

View file

@ -72,6 +72,8 @@ class FileRequest(object):
self.actionFindHashIds(params) self.actionFindHashIds(params)
elif cmd == "setHashfield": elif cmd == "setHashfield":
self.actionSetHashfield(params) self.actionSetHashfield(params)
elif cmd == "siteReload":
self.actionSiteReload(params)
elif cmd == "ping": elif cmd == "ping":
self.actionPing() self.actionPing()
else: else:
@ -314,6 +316,17 @@ class FileRequest(object):
peer.hashfield.replaceFromString(params["hashfield_raw"]) peer.hashfield.replaceFromString(params["hashfield_raw"])
self.response({"ok": "Updated"}) self.response({"ok": "Updated"})
def actionSiteReload(self, params):
if self.connection.ip != "127.0.0.1" and self.connection.ip != config.ip_external:
self.response({"error": "Only local host allowed"})
site = self.sites.get(params["site"])
site.content_manager.loadContent(params["inner_path"], add_bad_files=False)
site.storage.verifyFiles(quick_check=True)
site.updateWebsocket()
self.response({"ok": "Reloaded"})
# Send a simple Pong! answer # Send a simple Pong! answer
def actionPing(self): def actionPing(self):
self.response("Pong!") self.response("Pong!")

View file

@ -1,5 +1,6 @@
import logging import logging
import time import time
import sys
import gevent import gevent
@ -60,7 +61,11 @@ class Peer(object):
self.connection = None self.connection = None
try: try:
if self.site:
self.connection = self.site.connection_server.getConnection(self.ip, self.port) self.connection = self.site.connection_server.getConnection(self.ip, self.port)
else:
self.connection = sys.modules["main"].file_server.getConnection(self.ip, self.port)
except Exception, err: except Exception, err:
self.onConnectionError() self.onConnectionError()
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" % self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %

View file

@ -130,6 +130,15 @@ class Site:
if res is not True and res is not False: # Need downloading and file is allowed if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt file_threads.append(res) # Append evt
# Optionals files
if self.settings.get("autodownloadoptional"):
for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Start download and dont wait for finish, return the event
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Wait for includes download # Wait for includes download
include_threads = [] include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys(): for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
@ -212,6 +221,7 @@ class Site:
# Wait for peers # Wait for peers
if not self.peers: if not self.peers:
self.announce()
for wait in range(10): for wait in range(10):
time.sleep(5+wait) time.sleep(5+wait)
self.log.debug("Waiting for peers...") self.log.debug("Waiting for peers...")
@ -258,9 +268,6 @@ class Site:
self.log.debug("Fallback to old-style update") self.log.debug("Fallback to old-style update")
self.redownloadContents() self.redownloadContents()
if self.settings["own"]:
self.storage.verifyFiles(quick_check=True) # Check files (need for optional files)
else:
self.storage.checkFiles(quick_check=True) # Quick check and mark bad files based on file size self.storage.checkFiles(quick_check=True) # Quick check and mark bad files based on file size
changed, deleted = self.content_manager.loadContent("content.json") changed, deleted = self.content_manager.loadContent("content.json")

View file

@ -250,7 +250,7 @@ class SiteStorage:
return inner_path return inner_path
# Verify all files sha512sum using content.json # Verify all files sha512sum using content.json
def verifyFiles(self, quick_check=False): # Fast = using file size def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True):
bad_files = [] bad_files = []
if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first
@ -277,6 +277,7 @@ class SiteStorage:
if not ok: if not ok:
self.log.debug("[CHANGED] %s" % file_inner_path) self.log.debug("[CHANGED] %s" % file_inner_path)
if add_changed:
bad_files.append(file_inner_path) bad_files.append(file_inner_path)
# Optional files # Optional files
@ -288,6 +289,8 @@ class SiteStorage:
file_path = self.getPath(file_inner_path) file_path = self.getPath(file_inner_path)
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"]) self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"])
if add_optional:
bad_files.append(file_inner_path)
continue continue
if quick_check: if quick_check:
@ -301,6 +304,8 @@ class SiteStorage:
else: else:
self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"]) self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"])
optional_removed += 1 optional_removed += 1
if add_optional:
bad_files.append(file_inner_path)
self.log.debug("[OPTIONAL CHANGED] %s" % file_inner_path) self.log.debug("[OPTIONAL CHANGED] %s" % file_inner_path)
self.log.debug( self.log.debug(
@ -313,10 +318,15 @@ class SiteStorage:
# Check and try to fix site files integrity # Check and try to fix site files integrity
def checkFiles(self, quick_check=True): def checkFiles(self, quick_check=True):
s = time.time() s = time.time()
bad_files = self.verifyFiles(quick_check) bad_files = self.verifyFiles(
quick_check,
add_optional=self.site.settings.get("autodownloadoptional"),
add_changed=not self.site.settings.get("own") # Don't overwrite changed files if site owned
)
self.site.bad_files = {}
if bad_files: if bad_files:
for bad_file in bad_files: for bad_file in bad_files:
self.site.bad_files[bad_file] = self.site.bad_files.get("bad_file", 0) + 1 self.site.bad_files[bad_file] = 1
self.log.debug("Checked files in %.2fs... Quick:%s" % (time.time() - s, quick_check)) self.log.debug("Checked files in %.2fs... Quick:%s" % (time.time() - s, quick_check))
# Delete site's all file # Delete site's all file

View file

@ -138,6 +138,7 @@ class UiRequest(object):
headers = [] headers = []
headers.append(("Version", "HTTP/1.1")) headers.append(("Version", "HTTP/1.1"))
headers.append(("Connection", "Keep-Alive")) headers.append(("Connection", "Keep-Alive"))
headers.append(("Keep-Alive", "max=25, timeout=30"))
headers.append(("Access-Control-Allow-Origin", "*")) # Allow json access headers.append(("Access-Control-Allow-Origin", "*")) # Allow json access
if self.env["REQUEST_METHOD"] == "OPTIONS": if self.env["REQUEST_METHOD"] == "OPTIONS":
# Allow json access # Allow json access
@ -145,11 +146,11 @@ class UiRequest(object):
headers.append(("Access-Control-Allow-Credentials", "true")) headers.append(("Access-Control-Allow-Credentials", "true"))
cacheable_type = ( cacheable_type = (
content_type == "text/css" or content_type.startswith("image") or content_type == "text/css" or content_type.startswith("image") or content_type.startswith("video") or
self.env["REQUEST_METHOD"] == "OPTIONS" or content_type == "application/javascript" self.env["REQUEST_METHOD"] == "OPTIONS" or content_type == "application/javascript"
) )
if status == 200 and cacheable_type: # Cache Css, Js, Image files for 10min if status in (200, 206) and cacheable_type: # Cache Css, Js, Image files for 10min
headers.append(("Cache-Control", "public, max-age=600")) # Cache 10 min headers.append(("Cache-Control", "public, max-age=600")) # Cache 10 min
else: else:
headers.append(("Cache-Control", "no-cache, no-store, private, must-revalidate, max-age=0")) # No caching at all headers.append(("Cache-Control", "no-cache, no-store, private, must-revalidate, max-age=0")) # No caching at all
@ -380,7 +381,7 @@ class UiRequest(object):
range_end = int(re.match(".*?-([0-9]+)", range).group(1))+1 range_end = int(re.match(".*?-([0-9]+)", range).group(1))+1
else: else:
range_end = file_size range_end = file_size
extra_headers["Content-Length"] = range_end - range_start extra_headers["Content-Length"] = str(range_end - range_start)
extra_headers["Content-Range"] = "bytes %s-%s/%s" % (range_start, range_end-1, file_size) extra_headers["Content-Range"] = "bytes %s-%s/%s" % (range_start, range_end-1, file_size)
if range: if range:
status = 206 status = 206

View file

@ -152,6 +152,7 @@ class UiWebsocket(object):
if content: # Remove unnecessary data transfer if content: # Remove unnecessary data transfer
content = content.copy() content = content.copy()
content["files"] = len(content.get("files", {})) content["files"] = len(content.get("files", {}))
content["files_optional"] = len(content.get("files_optional", {}))
content["includes"] = len(content.get("includes", {})) content["includes"] = len(content.get("includes", {}))
if "sign" in content: if "sign" in content:
del(content["sign"]) del(content["sign"])

View file

@ -114,6 +114,18 @@ class WorkerManager:
continue # No peers found yet for the optional task continue # No peers found yet for the optional task
return task return task
def removeGoodFileTasks(self):
for task in self.tasks[:]:
if task["inner_path"] not in self.site.bad_files:
self.log.debug("No longer in bad_files, marking as good: %s" % task["inner_path"])
task["done"] = True
task["evt"].set(True)
self.tasks.remove(task)
if not self.tasks:
self.started_task_num = 0
self.site.updateWebsocket()
# New peers added to site # New peers added to site
def onPeers(self): def onPeers(self):
self.startWorkers() self.startWorkers()

View file

@ -226,18 +226,28 @@ class Actions(object):
global file_server global file_server
from Site import SiteManager from Site import SiteManager
from File import FileServer # We need fileserver to handle incoming file requests from File import FileServer # We need fileserver to handle incoming file requests
from Peer import Peer
logging.info("Creating FileServer....") logging.info("Creating FileServer....")
file_server = FileServer() file_server = FileServer()
file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity
file_server.openport() file_server.openport()
site = SiteManager.site_manager.list()[address] site = SiteManager.site_manager.list()[address]
site.settings["serving"] = True # Serving the site even if its disabled site.settings["serving"] = True # Serving the site even if its disabled
# Notify local client on new content
if config.ip_external:
logging.info("Sending siteReload")
my_peer = Peer(config.ip_external, config.fileserver_port)
logging.info(my_peer.request("siteReload", {"site": site.address, "inner_path": inner_path}))
if peer_ip: # Announce ip specificed if peer_ip: # Announce ip specificed
site.addPeer(peer_ip, peer_port) site.addPeer(peer_ip, peer_port)
else: # Just ask the tracker else: # Just ask the tracker
logging.info("Gathering peers from tracker") logging.info("Gathering peers from tracker")
site.announce() # Gather peers site.announce() # Gather peers
published = site.publish(20, inner_path) # Push to 20 peers published = site.publish(20, inner_path) # Push to 20 peers
if published > 0: if published > 0:
time.sleep(3) time.sleep(3)