oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

no need to specify content.json priority, its always downloaded first, peer broken after 3 hash failes, extra priority for css and js files

Browse source
This commit is contained in:
HelloZeroNet 2015-01-14 22:57:43 +01:00
parent 776b572c82
commit bcd0c0025a
4 changed files with 22 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -42,6 +42,7 @@ Linux (Debian):
## How can I create a ZeroNet site?
Shut down zeronet.py if you are running it already
```
$ zeronet.py siteCreate
...

2
src/Site/Site.py
View file

@ -189,7 +189,7 @@ class Site:
self.log.debug("Need content.json first")
self.announce()
if inner_path != "content.json": # Prevent double download
task = self.worker_manager.addTask("content.json", peer, priority=99999)
task = self.worker_manager.addTask("content.json", peer)
task.get()
self.loadContent()
if not self.content: return False

4
src/Worker/Worker.py
View file

@ -24,8 +24,10 @@ class Worker:
if task["workers_num"] > 0: # Wait a bit if someone already working on it
self.manager.log.debug("%s: Someone already working on %s, sleeping 1 sec..." % (self.key, task["inner_path"]))
time.sleep(1)
self.manager.log.debug("%s: %s, task done after sleep: %s" % (self.key, task["inner_path"], task["done"]))
if task["done"] == False:
if not task["time_started"]: task["time_started"] = time.time() # Task started now
self.task = task
task["workers_num"] += 1
buff = self.peer.getFile(task["site"].address, task["inner_path"])
@ -49,7 +51,7 @@ class Worker:
else: # Hash failed
self.task = None
self.peer.hash_failed += 1
if self.peer.hash_failed > 5: # Broken peer
if self.peer.hash_failed >= 3: # Broken peer
break
task["workers_num"] -= 1
self.manager.log.error("%s: Hash failed: %s" % (self.key, task["inner_path"]))

22
src/Worker/WorkerManager.py
View file

@ -8,7 +8,7 @@ class WorkerManager:
def __init__(self, site):
self.site = site
self.workers = {} # Key: ip:port, Value: Worker.Worker
self.tasks = [] # {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_start": time.time(), "peers": peers, "priority": 0}
self.tasks = [] # {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_started": None, "time_added": time.time(), "peers": peers, "priority": 0}
self.log = logging.getLogger("WorkerManager:%s" % self.site.address_short)
self.process_taskchecker = gevent.spawn(self.checkTasks)
@ -20,31 +20,33 @@ class WorkerManager:
if not self.tasks: continue
tasks = self.tasks[:] # Copy it so removing elements wont cause any problem
for task in tasks:
if time.time() >= task["time_start"]+60: # Task timed out
self.log.debug("Cleaning up task: %s" % task)
if (task["time_started"] and time.time() >= task["time_started"]+60) or (time.time() >= task["time_added"]+60 and not self.workers): # Task taking too long time, kill it
self.log.debug("Timeout, Cleaning up task: %s" % task)
# Clean up workers
workers = self.findWorkers(task)
for worker in workers:
worker.stop()
# Remove task
self.failTask(task)
elif time.time() >= task["time_start"]+15: # Task taking long time
self.log.debug("Task taking long time, find more peers: %s" % task["inner_path"])
elif (task["time_started"] and time.time() >= task["time_started"]+15) or not self.workers: # Task started more than 15 sec ago or no workers
self.log.debug("Task taking more than 15 secs, find more peers: %s" % task["inner_path"])
task["site"].announce() # Find more peers
if task["peers"]: # Release the peer olck
self.log.debug("Task peer lock release: %s" % task["inner_path"])
task["peers"] = []
self.startWorkers()
continue # One reannounce per loop
break # One reannounce per loop
# Tasks sorted by this
def taskSorter(self, task):
if task["inner_path"] == "content.json": return 9999 # Content.json always prority
if task["inner_path"] == "index.html": return 9998 # index.html also important
return task["priority"]-task["workers_num"] # Prefer more priority and less workers
priority = task["priority"]
if task["inner_path"].endswith(".js") or task["inner_path"].endswith(".css"): priority += 1 # download js and css files first
return priority-task["workers_num"] # Prefer more priority and less workers
# Returns the next free or less worked task
@ -105,7 +107,7 @@ class WorkerManager:
peers = [peer] # Only download from this peer
else:
peers = None
task = {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_start": time.time(), "peers": peers, "priority": priority}
task = {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_added": time.time(), "time_started": None, "peers": peers, "priority": priority}
self.tasks.append(task)
self.log.debug("New task: %s" % task)
self.startWorkers()