oci/zeronet
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Avoid write race on same file

Browse source
This commit is contained in:
shortcutme 2019-11-30 02:04:59 +01:00
parent e1dc29c374
commit 1c587bde25
No known key found for this signature in database
GPG key ID: 5B63BAE6CB9613AE
2 changed files with 13 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/Worker/Worker.py
View file

@ -107,7 +107,9 @@ class Worker(object):
if self.manager.started_task_num < 50 or config.verbose: if self.manager.started_task_num < 50 or config.verbose:
self.manager.log.debug("%s: Verify correct: %s" % (self.key, task["inner_path"])) self.manager.log.debug("%s: Verify correct: %s" % (self.key, task["inner_path"]))
write_error = None write_error = None
if correct is True and task["done"] is False: # Save if changed and task not done yet task_finished = False
if correct is True and task["locked"] is False: # Save if changed and task not done yet
task["locked"] = True
buff.seek(0) buff.seek(0)
try: try:
site.storage.write(task["inner_path"], buff) site.storage.write(task["inner_path"], buff)
@ -115,8 +117,13 @@ class Worker(object):
except Exception as err: except Exception as err:
self.manager.log.error("%s: Error writing: %s (%s)" % (self.key, task["inner_path"], err)) self.manager.log.error("%s: Error writing: %s (%s)" % (self.key, task["inner_path"], err))
write_error = err write_error = err
task_finished = True
if task["done"] is False: if correct is None and task["locked"] is False: # Mark as done if same file
task["locked"] = True
task_finished = True
if task_finished and not task["done"]:
if write_error: if write_error:
self.manager.failTask(task) self.manager.failTask(task)
self.num_failed += 1 self.num_failed += 1

6
src/Worker/WorkerManager.py
View file

@ -21,7 +21,7 @@ class WorkerManager(object):
self.tasks = WorkerTaskManager() self.tasks = WorkerTaskManager()
self.next_task_id = 1 self.next_task_id = 1
# {"id": 1, "evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "optional_hash_id": None, # {"id": 1, "evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "optional_hash_id": None,
# "time_started": None, "time_added": time.time(), "peers": peers, "priority": 0, "failed": peer_ids} # "time_started": None, "time_added": time.time(), "peers": peers, "priority": 0, "failed": peer_ids, "locked": False}
self.started_task_num = 0 # Last added task num self.started_task_num = 0 # Last added task num
self.asked_peers = [] self.asked_peers = []
self.running = True self.running = True
@ -124,6 +124,8 @@ class WorkerManager(object):
continue # Peer already tried to solve this, but failed continue # Peer already tried to solve this, but failed
if task["optional_hash_id"] and task["peers"] is None: if task["optional_hash_id"] and task["peers"] is None:
continue # No peers found yet for the optional task continue # No peers found yet for the optional task
if task["done"]:
continue
return task return task
def removeSolvedFileTasks(self, mark_as_good=True): def removeSolvedFileTasks(self, mark_as_good=True):
@ -498,7 +500,7 @@ class WorkerManager(object):
task = { task = {
"id": self.next_task_id, "evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "id": self.next_task_id, "evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False,
"optional_hash_id": optional_hash_id, "time_added": time.time(), "time_started": None, "optional_hash_id": optional_hash_id, "time_added": time.time(), "time_started": None, "locked": False,
"time_action": None, "peers": peers, "priority": priority, "failed": [], "size": size "time_action": None, "peers": peers, "priority": priority, "failed": [], "size": size
} }