diff --git a/plugins/OptionalManager/ContentDbPlugin.py b/plugins/OptionalManager/ContentDbPlugin.py
new file mode 100644
index 00000000..af0681c7
--- /dev/null
+++ b/plugins/OptionalManager/ContentDbPlugin.py
@@ -0,0 +1,379 @@
+import time
+import collections
+import itertools
+import re
+
+import gevent
+
+from util import helper
+from Plugin import PluginManager
+from Config import config
+
+if "content_db" not in locals().keys(): # To keep between module reloads
+ content_db = None
+
+
+@PluginManager.registerTo("ContentDb")
+class ContentDbPlugin(object):
+ def __init__(self, *args, **kwargs):
+ global content_db
+ content_db = self
+ self.filled = {} # Site addresses that already filled from content.json
+ self.need_filling = False # file_optional table just created, fill data from content.json files
+ self.time_peer_numbers_updated = 0
+ self.my_optional_files = {} # Last 50 site_address/inner_path called by fileWrite (auto-pinning these files)
+ self.optional_files = collections.defaultdict(dict)
+ self.optional_files_loading = False
+ helper.timer(60 * 5, self.checkOptionalLimit)
+ super(ContentDbPlugin, self).__init__(*args, **kwargs)
+
+ def getSchema(self):
+ schema = super(ContentDbPlugin, self).getSchema()
+
+ # Need file_optional table
+ schema["tables"]["file_optional"] = {
+ "cols": [
+ ["file_id", "INTEGER PRIMARY KEY UNIQUE NOT NULL"],
+ ["site_id", "INTEGER REFERENCES site (site_id) ON DELETE CASCADE"],
+ ["inner_path", "TEXT"],
+ ["hash_id", "INTEGER"],
+ ["size", "INTEGER"],
+ ["peer", "INTEGER DEFAULT 0"],
+ ["uploaded", "INTEGER DEFAULT 0"],
+ ["is_downloaded", "INTEGER DEFAULT 0"],
+ ["is_pinned", "INTEGER DEFAULT 0"],
+ ["time_added", "INTEGER DEFAULT 0"],
+ ["time_downloaded", "INTEGER DEFAULT 0"],
+ ["time_accessed", "INTEGER DEFAULT 0"]
+ ],
+ "indexes": [
+ "CREATE UNIQUE INDEX file_optional_key ON file_optional (site_id, inner_path)",
+ "CREATE INDEX deletable ON file_optional (peer, is_downloaded) WHERE peer > 10"
+ ],
+ "schema_changed": 11
+ }
+
+ return schema
+
+ def initSite(self, site):
+ super(ContentDbPlugin, self).initSite(site)
+ if self.need_filling:
+ self.fillTableFileOptional(site)
+ if not self.optional_files_loading:
+ gevent.spawn_later(1, self.loadFilesOptional)
+ self.optional_files_loading = True
+
+ def checkTables(self):
+ changed_tables = super(ContentDbPlugin, self).checkTables()
+ if "file_optional" in changed_tables:
+ self.need_filling = True
+ return changed_tables
+
+ # Load optional files ending
+ def loadFilesOptional(self):
+ s = time.time()
+ num = 0
+ total = 0
+ total_downloaded = 0
+ res = content_db.execute("SELECT site_id, inner_path, size, is_downloaded FROM file_optional")
+ site_sizes = collections.defaultdict(lambda: collections.defaultdict(int))
+ for row in res:
+ self.optional_files[row["site_id"]][row["inner_path"][-8:]] = 1
+ num += 1
+
+ # Update site size stats
+ site_sizes[row["site_id"]]["size_optional"] += row["size"]
+ if row["is_downloaded"]:
+ site_sizes[row["site_id"]]["optional_downloaded"] += row["size"]
+
+ # Site site size stats to sites.json settings
+ site_ids_reverse = {val: key for key, val in self.site_ids.iteritems()}
+ for site_id, stats in site_sizes.iteritems():
+ site_address = site_ids_reverse.get(site_id)
+ if not site_address:
+ self.log.error("Not found site_id: %s" % site_id)
+ continue
+ site = self.sites[site_address]
+ site.settings["size_optional"] = stats["size_optional"]
+ site.settings["optional_downloaded"] = stats["optional_downloaded"]
+ total += stats["size_optional"]
+ total_downloaded += stats["optional_downloaded"]
+
+ self.log.debug(
+ "Loaded %s optional files: %.2fMB, downloaded: %.2fMB in %.3fs" %
+ (num, float(total) / 1024 / 1024, float(total_downloaded) / 1024 / 1024, time.time() - s)
+ )
+
+ if self.need_filling and self.getOptionalLimitBytes() < total_downloaded:
+ limit_bytes = self.getOptionalLimitBytes()
+ limit_new = round((float(total_downloaded) / 1024 / 1024 / 1024) * 1.1, 2) # Current limit + 10%
+ self.log.debug(
+ "First startup after update and limit is smaller than downloaded files size (%.2fGB), increasing it from %.2fGB to %.2fGB" %
+ (float(total_downloaded) / 1024 / 1024 / 1024, float(limit_bytes) / 1024 / 1024 / 1024, limit_new)
+ )
+ config.saveValue("optional_limit", limit_new)
+ config.optional_limit = str(limit_new)
+
+ # Predicts if the file is optional
+ def isOptionalFile(self, site_id, inner_path):
+ return self.optional_files[site_id].get(inner_path[-8:])
+
+ # Fill file_optional table with optional files found in sites
+ def fillTableFileOptional(self, site):
+ s = time.time()
+ site_id = self.site_ids.get(site.address)
+ if not site_id:
+ return False
+ cur = self.getCursor()
+ cur.execute("BEGIN")
+ res = cur.execute("SELECT * FROM content WHERE size_files_optional > 0 AND site_id = %s" % site_id)
+ num = 0
+ for row in res.fetchall():
+ content = site.content_manager.contents[row["inner_path"]]
+ try:
+ num += self.setContentFilesOptional(site, row["inner_path"], content, cur=cur)
+ except Exception, err:
+ self.log.error("Error loading %s into file_optional: %s" % (row["inner_path"], err))
+ cur.execute("COMMIT")
+ cur.close()
+
+ # Set my files to pinned
+ from User import UserManager
+ user = UserManager.user_manager.get()
+ if not user:
+ user = UserManager.user_manager.create()
+ auth_address = user.getAuthAddress(site.address)
+ self.execute(
+ "UPDATE file_optional SET is_pinned = 1 WHERE site_id = :site_id AND inner_path LIKE :inner_path",
+ {"site_id": site_id, "inner_path": "%%/%s/%%" % auth_address}
+ )
+
+ self.log.debug(
+ "Filled file_optional table for %s in %.3fs (loaded: %s, is_pinned: %s)" %
+ (site.address, time.time() - s, num, self.cur.cursor.rowcount)
+ )
+ self.filled[site.address] = True
+
+ def setContentFilesOptional(self, site, content_inner_path, content, cur=None):
+ if not cur:
+ cur = self
+ cur.execute("BEGIN")
+
+ num = 0
+ site_id = self.site_ids[site.address]
+ content_inner_dir = helper.getDirname(content_inner_path)
+ for relative_inner_path, file in content.get("files_optional", {}).iteritems():
+ file_inner_path = content_inner_dir + relative_inner_path
+ hash_id = int(file["sha512"][0:4], 16)
+ if hash_id in site.content_manager.hashfield:
+ is_downloaded = 1
+ else:
+ is_downloaded = 0
+ if site.address + "/" + file_inner_path in self.my_optional_files:
+ is_pinned = 1
+ else:
+ is_pinned = 0
+ cur.insertOrUpdate("file_optional", {
+ "hash_id": hash_id,
+ "size": int(file["size"]),
+ "is_pinned": is_pinned
+ }, {
+ "site_id": site_id,
+ "inner_path": file_inner_path
+ }, oninsert={
+ "time_added": int(time.time()),
+ "time_downloaded": int(time.time()) if is_downloaded else 0,
+ "is_downloaded": is_downloaded,
+ "peer": is_downloaded
+ })
+ self.optional_files[site_id][file_inner_path[-8:]] = 1
+ num += 1
+
+ if cur == self:
+ cur.execute("END")
+
+ return num
+
+ def setContent(self, site, inner_path, content, size=0):
+ super(ContentDbPlugin, self).setContent(site, inner_path, content, size=size)
+ old_content = site.content_manager.contents.get(inner_path, {})
+ if (not self.need_filling or self.filled.get(site.address)) and "files_optional" in content or "files_optional" in old_content:
+ self.setContentFilesOptional(site, inner_path, content)
+ # Check deleted files
+ if old_content:
+ old_files = old_content.get("files_optional", {}).keys()
+ new_files = content.get("files_optional", {}).keys()
+ content_inner_dir = helper.getDirname(inner_path)
+ deleted = [content_inner_dir + key for key in old_files if key not in new_files]
+ if deleted:
+ site_id = self.site_ids[site.address]
+ self.execute("DELETE FROM file_optional WHERE ?", {"site_id": site_id, "inner_path": deleted})
+
+ def deleteContent(self, site, inner_path):
+ content = site.content_manager.contents.get(inner_path)
+ if content and "files_optional" in content:
+ site_id = self.site_ids[site.address]
+ content_inner_dir = helper.getDirname(inner_path)
+ optional_inner_paths = [
+ content_inner_dir + relative_inner_path
+ for relative_inner_path in content.get("files_optional", {}).keys()
+ ]
+ self.execute("DELETE FROM file_optional WHERE ?", {"site_id": site_id, "inner_path": optional_inner_paths})
+ super(ContentDbPlugin, self).deleteContent(site, inner_path)
+
+ def updatePeerNumbers(self):
+ s = time.time()
+ num_file = 0
+ num_updated = 0
+ num_site = 0
+ for site in self.sites.values():
+ if not site.content_manager.has_optional_files:
+ continue
+ has_updated_hashfield = next((
+ peer
+ for peer in site.peers.itervalues()
+ if peer.has_hashfield and peer.hashfield.time_changed > self.time_peer_numbers_updated
+ ), None)
+
+ if not has_updated_hashfield and site.content_manager.hashfield.time_changed < self.time_peer_numbers_updated:
+ continue
+
+ hashfield_peers = itertools.chain.from_iterable(
+ peer.hashfield.storage
+ for peer in site.peers.itervalues()
+ if peer.has_hashfield
+ )
+ peer_nums = collections.Counter(
+ itertools.chain(
+ hashfield_peers,
+ site.content_manager.hashfield
+ )
+ )
+
+ site_id = self.site_ids[site.address]
+ if not site_id:
+ continue
+
+ res = self.execute("SELECT file_id, hash_id, peer FROM file_optional WHERE ?", {"site_id": site_id})
+ updates = {}
+ for row in res:
+ peer_num = peer_nums.get(row["hash_id"], 0)
+ if peer_num != row["peer"]:
+ updates[row["file_id"]] = peer_num
+
+ self.execute("BEGIN")
+ for file_id, peer_num in updates.iteritems():
+ self.execute("UPDATE file_optional SET peer = ? WHERE file_id = ?", (peer_num, file_id))
+ self.execute("END")
+
+ num_updated += len(updates)
+ num_file += len(peer_nums)
+ num_site += 1
+
+ self.time_peer_numbers_updated = time.time()
+ self.log.debug("%s/%s peer number for %s site updated in %.3fs" % (num_updated, num_file, num_site, time.time() - s))
+
+ def queryDeletableFiles(self):
+ # First return the files with atleast 10 seeder and not accessed in last weed
+ query = """
+ SELECT * FROM file_optional
+ WHERE peer > 10 AND is_downloaded = 1 AND is_pinned = 0
+ ORDER BY time_accessed < %s DESC, uploaded / size
+ """ % int(time.time() - 60 * 60 * 7)
+ limit_start = 0
+ while 1:
+ num = 0
+ res = self.execute("%s LIMIT %s, 50" % (query, limit_start))
+ for row in res:
+ yield row
+ num += 1
+ if num < 50:
+ break
+ limit_start += 50
+
+ self.log.debug("queryDeletableFiles returning less-seeded files")
+
+ # Then return files less seeder but still not accessed in last week
+ query = """
+ SELECT * FROM file_optional
+ WHERE is_downloaded = 1 AND peer <= 10 AND is_pinned = 0
+ ORDER BY peer DESC, time_accessed < %s DESC, uploaded / size
+ """ % int(time.time() - 60 * 60 * 7)
+ limit_start = 0
+ while 1:
+ num = 0
+ res = self.execute("%s LIMIT %s, 50" % (query, limit_start))
+ for row in res:
+ yield row
+ num += 1
+ if num < 50:
+ break
+ limit_start += 50
+
+ self.log.debug("queryDeletableFiles returning everyting")
+
+ # At the end return all files
+ query = """
+ SELECT * FROM file_optional
+ WHERE is_downloaded = 1 AND peer <= 10 AND is_pinned = 0
+ ORDER BY peer DESC, time_accessed, uploaded / size
+ """
+ limit_start = 0
+ while 1:
+ num = 0
+ res = self.execute("%s LIMIT %s, 50" % (query, limit_start))
+ for row in res:
+ yield row
+ num += 1
+ if num < 50:
+ break
+ limit_start += 50
+
+ def getOptionalLimitBytes(self):
+ if config.optional_limit.endswith("%"):
+ limit_percent = float(re.sub("[^0-9.]", "", config.optional_limit))
+ limit_bytes = helper.getFreeSpace() * (limit_percent / 100)
+ else:
+ limit_bytes = float(re.sub("[^0-9.]", "", config.optional_limit)) * 1024 * 1024 * 1024
+ return limit_bytes
+
+ def checkOptionalLimit(self, limit=None):
+ if not limit:
+ limit = self.getOptionalLimitBytes()
+
+ size = self.execute("SELECT SUM(size) FROM file_optional WHERE is_downloaded = 1 AND is_pinned = 0").fetchone()[0]
+ if not size:
+ size = 0
+ need_delete = size - limit
+ self.log.debug("Optional size: %.1fMB/%.1fMB" % (float(size) / 1024 / 1024, float(limit) / 1024 / 1024))
+ if need_delete <= 0:
+ return False
+
+ self.updatePeerNumbers()
+
+ site_ids_reverse = {val: key for key, val in self.site_ids.iteritems()}
+ deleted_file_ids = []
+ for row in self.queryDeletableFiles():
+ site_address = site_ids_reverse.get(row["site_id"])
+ site = self.sites.get(site_address)
+ if not site:
+ self.log.error("No site found for id: %s" % row["site_id"])
+ continue
+ site.log.debug("Deleting %s %.3f MB left" % (row["inner_path"], float(need_delete) / 1024 / 1024))
+ deleted_file_ids.append(row["file_id"])
+ try:
+ site.content_manager.optionalRemove(row["inner_path"], row["hash_id"], row["size"])
+ site.storage.delete(row["inner_path"])
+ need_delete -= row["size"]
+ except Exception, err:
+ site.log.error("Error deleting %s: %s" % (row["inner_path"], err))
+
+ if need_delete <= 0:
+ break
+
+ cur = self.getCursor()
+ cur.execute("BEGIN")
+ for file_id in deleted_file_ids:
+ cur.execute("UPDATE file_optional SET is_downloaded = 0, is_pinned = 0, peer = peer - 1 WHERE ?", {"file_id": file_id})
+ cur.execute("COMMIT")
+ cur.close()
diff --git a/plugins/OptionalManager/OptionalManagerPlugin.py b/plugins/OptionalManager/OptionalManagerPlugin.py
new file mode 100644
index 00000000..687c87af
--- /dev/null
+++ b/plugins/OptionalManager/OptionalManagerPlugin.py
@@ -0,0 +1,117 @@
+import time
+import collections
+
+from util import helper
+from Plugin import PluginManager
+import ContentDbPlugin
+
+
+def processAccessLog():
+ if access_log:
+ content_db = ContentDbPlugin.content_db
+ now = int(time.time())
+ num = 0
+ for site_id in access_log:
+ content_db.execute(
+ "UPDATE file_optional SET time_accessed = %s WHERE ?" % now,
+ {"site_id": site_id, "inner_path": access_log[site_id].keys()}
+ )
+ num += len(access_log[site_id])
+ access_log.clear()
+
+
+def processRequestLog():
+ if request_log:
+ content_db = ContentDbPlugin.content_db
+ cur = content_db.getCursor()
+ num = 0
+ cur.execute("BEGIN")
+ for site_id in request_log:
+ for inner_path, uploaded in request_log[site_id].iteritems():
+ content_db.execute(
+ "UPDATE file_optional SET uploaded = uploaded + %s WHERE ?" % uploaded,
+ {"site_id": site_id, "inner_path": inner_path}
+ )
+ num += 1
+ cur.execute("END")
+ request_log.clear()
+
+
+if "access_log" not in locals().keys(): # To keep between module reloads
+ access_log = collections.defaultdict(dict) # {site_id: {inner_path1: 1, inner_path2: 1...}}
+ request_log = collections.defaultdict(lambda: collections.defaultdict(int)) # {site_id: {inner_path1: 1, inner_path2: 1...}}
+ helper.timer(61, processAccessLog)
+ helper.timer(60, processRequestLog)
+
+
+@PluginManager.registerTo("WorkerManager")
+class WorkerManagerPlugin(object):
+ def doneTask(self, task):
+ if task["optional_hash_id"]:
+ content_db = self.site.content_manager.contents.db
+ content_db.executeDelayed(
+ "UPDATE file_optional SET time_downloaded = :now, is_downloaded = 1, peer = peer + 1 WHERE site_id = :site_id AND inner_path = :inner_path",
+ {"now": int(time.time()), "site_id": content_db.site_ids[self.site.address], "inner_path": task["inner_path"]}
+ )
+
+ super(WorkerManagerPlugin, self).doneTask(task)
+
+ if task["optional_hash_id"] and not self.tasks:
+ content_db.processDelayed()
+
+
+@PluginManager.registerTo("UiRequest")
+class UiRequestPlugin(object):
+ def parsePath(self, path):
+ global access_log
+ path_parts = super(UiRequestPlugin, self).parsePath(path)
+ if path_parts:
+ site_id = ContentDbPlugin.content_db.site_ids.get(path_parts["request_address"])
+ if site_id:
+ if ContentDbPlugin.content_db.isOptionalFile(site_id, path_parts["inner_path"]):
+ access_log[site_id][path_parts["inner_path"]] = 1
+ return path_parts
+
+
+@PluginManager.registerTo("FileRequest")
+class FileRequestPlugin(object):
+ def actionGetFile(self, params):
+ stats = super(FileRequestPlugin, self).actionGetFile(params)
+ self.recordFileRequest(params["site"], params["inner_path"], stats)
+ return stats
+
+ def actionStreamFile(self, params):
+ stats = super(FileRequestPlugin, self).actionStreamFile(params)
+ self.recordFileRequest(params["site"], params["inner_path"], stats)
+ return stats
+
+ def recordFileRequest(self, site_address, inner_path, stats):
+ if not stats:
+ # Only track the last request of files
+ return False
+ site_id = ContentDbPlugin.content_db.site_ids[site_address]
+ if site_id and ContentDbPlugin.content_db.isOptionalFile(site_id, inner_path):
+ request_log[site_id][inner_path] += stats["bytes_sent"]
+
+
+@PluginManager.registerTo("Site")
+class SitePlugin(object):
+ def isDownloadable(self, inner_path):
+ is_downloadable = super(SitePlugin, self).isDownloadable(inner_path)
+ if is_downloadable:
+ return is_downloadable
+
+ for path in self.settings.get("optional_help", {}).iterkeys():
+ if inner_path.startswith(path):
+ return True
+
+ return False
+
+
+@PluginManager.registerTo("ConfigPlugin")
+class ConfigPlugin(object):
+ def createArguments(self):
+ group = self.parser.add_argument_group("OptionalManager plugin")
+ group.add_argument('--optional_limit', help='Limit total size of optional files', default="10%", metavar="GB or free space %")
+
+ return super(ConfigPlugin, self).createArguments()
diff --git a/plugins/OptionalManager/UiWebsocketPlugin.py b/plugins/OptionalManager/UiWebsocketPlugin.py
new file mode 100644
index 00000000..ddce352b
--- /dev/null
+++ b/plugins/OptionalManager/UiWebsocketPlugin.py
@@ -0,0 +1,257 @@
+import re
+import time
+import cgi
+
+import gevent
+
+from Plugin import PluginManager
+from Config import config
+from util import helper
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+ def __init__(self, *args, **kwargs):
+ self.time_peer_numbers_updated = 0
+ super(UiWebsocketPlugin, self).__init__(*args, **kwargs)
+
+ def actionFileWrite(self, to, inner_path, *args, **kwargs):
+ super(UiWebsocketPlugin, self).actionFileWrite(to, inner_path, *args, **kwargs)
+
+ # Add file to content.db and set it as pinned
+ content_db = self.site.content_manager.contents.db
+ content_db.my_optional_files[self.site.address + "/" + inner_path] = time.time()
+ if len(content_db.my_optional_files) > 50: # Keep only last 50
+ oldest_key = min(
+ content_db.my_optional_files.iterkeys(),
+ key=(lambda key: content_db.my_optional_files[key])
+ )
+ del content_db.my_optional_files[oldest_key]
+
+ def updatePeerNumbers(self):
+ content_db = self.site.content_manager.contents.db
+ content_db.updatePeerNumbers()
+ self.site.updateWebsocket(peernumber_updated=True)
+
+ # Optional file functions
+
+ def actionOptionalFileList(self, to, address=None, orderby="time_downloaded DESC", limit=10):
+ if not address:
+ address = self.site.address
+
+ # Update peer numbers if necessary
+ content_db = self.site.content_manager.contents.db
+ if time.time() - content_db.time_peer_numbers_updated > 60 * 1 and time.time() - self.time_peer_numbers_updated > 60 * 5:
+ # Start in new thread to avoid blocking
+ self.time_peer_numbers_updated = time.time()
+ gevent.spawn(self.updatePeerNumbers)
+
+ if address != self.site.address and "ADMIN" not in self.site.settings["permissions"]:
+ return self.response(to, "optionalSiteInfo not allowed on this site")
+
+ if not all([re.match("^[a-z_*/+-]+( DESC| ASC|)$", part.strip()) for part in orderby.split(",")]):
+ return self.response(to, "Invalid order_by")
+
+ if type(limit) != int:
+ return self.response(to, "Invalid limit")
+
+ back = []
+ content_db = self.site.content_manager.contents.db
+ site_id = content_db.site_ids[address]
+ query = "SELECT * FROM file_optional WHERE site_id = %s AND is_downloaded = 1 ORDER BY %s LIMIT %s" % (site_id, orderby, limit)
+ for row in content_db.execute(query):
+ back.append(dict(row))
+ self.response(to, back)
+
+ def actionOptionalFileInfo(self, to, inner_path):
+ content_db = self.site.content_manager.contents.db
+ site_id = content_db.site_ids[self.site.address]
+
+ # Update peer numbers if necessary
+ if time.time() - content_db.time_peer_numbers_updated > 60 * 1 and time.time() - self.time_peer_numbers_updated > 60 * 5:
+ # Start in new thread to avoid blocking
+ self.time_peer_numbers_updated = time.time()
+ gevent.spawn(self.updatePeerNumbers)
+
+ query = "SELECT * FROM file_optional WHERE site_id = :site_id AND inner_path = :inner_path LIMIT 1"
+ res = content_db.execute(query, {"site_id": site_id, "inner_path": inner_path})
+ row = next(res, None)
+ if row:
+ self.response(to, dict(row))
+ else:
+ self.response(to, None)
+
+ def setPin(self, inner_path, is_pinned, address=None):
+ if not address:
+ address = self.site.address
+
+ if not self.hasSitePermission(address):
+ return {"error": "Forbidden"}
+
+ site = self.server.sites[address]
+
+ content_db = site.content_manager.contents.db
+ site_id = content_db.site_ids[site.address]
+ content_db.execute("UPDATE file_optional SET is_pinned = %s WHERE ?" % is_pinned, {"site_id": site_id, "inner_path": inner_path})
+
+ return "ok"
+
+ def actionOptionalFilePin(self, to, inner_path, address=None):
+ back = self.setPin(inner_path, 1, address)
+ if back == "ok":
+ self.cmd("notification", ["done", "Pinned %s files" % len(inner_path) if type(inner_path) is list else 1, 5000])
+ self.response(to, back)
+
+ def actionOptionalFileUnpin(self, to, inner_path, address=None):
+ back = self.setPin(inner_path, 0, address)
+ if back == "ok":
+ self.cmd("notification", ["done", "Removed pin from %s files" % len(inner_path) if type(inner_path) is list else 1, 5000])
+ self.response(to, back)
+
+ def actionOptionalFileDelete(self, to, inner_path, address=None):
+ if not address:
+ address = self.site.address
+
+ if not self.hasSitePermission(address):
+ return self.response(to, {"error": "Forbidden"})
+
+ site = self.server.sites[address]
+
+ content_db = site.content_manager.contents.db
+ site_id = content_db.site_ids[site.address]
+
+ res = content_db.execute("SELECT * FROM file_optional WHERE ? LIMIT 1", {"site_id": site_id, "inner_path": inner_path})
+ row = next(res, None)
+
+ if not row:
+ return self.response(to, {"error": "Not found in content.db"})
+
+ removed = site.content_manager.optionalRemove(inner_path, row["hash_id"], row["size"])
+ # if not removed:
+ # return self.response(to, {"error": "Not found in hash_id: %s" % row["hash_id"]})
+
+ content_db.execute("UPDATE file_optional SET is_downloaded = 0, is_pinned = 0, peer = peer - 1 WHERE ?", {"site_id": site_id, "inner_path": inner_path})
+
+ try:
+ site.storage.delete(inner_path)
+ except Exception, err:
+ return self.response(to, {"error": "File delete error: %s" % err})
+
+ self.response(to, "ok")
+
+
+ # Limit functions
+
+ def actionOptionalLimitStats(self, to):
+ if "ADMIN" not in self.site.settings["permissions"]:
+ return self.response(to, "Forbidden")
+
+ back = {}
+ back["limit"] = config.optional_limit
+ back["used"] = self.site.content_manager.contents.db.execute(
+ "SELECT SUM(size) FROM file_optional WHERE is_downloaded = 1 AND is_pinned = 0"
+ ).fetchone()[0]
+ back["free"] = helper.getFreeSpace()
+
+ self.response(to, back)
+
+ def actionOptionalLimitSet(self, to, limit):
+ if "ADMIN" not in self.site.settings["permissions"]:
+ return self.response(to, {"error": "Forbidden"})
+ config.optional_limit = re.sub("\.0+$", "", limit) # Remove unnecessary digits from end
+ config.saveValue("optional_limit", limit)
+ self.response(to, "ok")
+
+ # Distribute help functions
+
+ def actionOptionalHelpList(self, to, address=None):
+ if not address:
+ address = self.site.address
+
+ if not self.hasSitePermission(address):
+ return self.response(to, {"error": "Forbidden"})
+
+ site = self.server.sites[address]
+
+ self.response(to, site.settings.get("optional_help", {}))
+
+ def actionOptionalHelp(self, to, directory, title, address=None):
+ if not address:
+ address = self.site.address
+
+ if not self.hasSitePermission(address):
+ return self.response(to, {"error": "Forbidden"})
+
+ site = self.server.sites[address]
+ content_db = site.content_manager.contents.db
+ site_id = content_db.site_ids[address]
+
+ if "optional_help" not in site.settings:
+ site.settings["optional_help"] = {}
+
+ stats = content_db.execute(
+ "SELECT COUNT(*) AS num, SUM(size) AS size FROM file_optional WHERE site_id = :site_id AND inner_path LIKE :inner_path",
+ {"site_id": site_id, "inner_path": directory + "%"}
+ ).fetchone()
+ stats = dict(stats)
+
+ if not stats["size"]:
+ stats["size"] = 0
+ if not stats["num"]:
+ stats["num"] = 0
+
+ self.cmd("notification", [
+ "done",
+ "You started to help distribute %s.
Directory: %s" %
+ (cgi.escape(title), cgi.escape(directory)),
+ 10000
+ ])
+
+ site.settings["optional_help"][directory] = title
+
+ self.response(to, dict(stats))
+
+ def actionOptionalHelpRemove(self, to, directory, address=None):
+ if not address:
+ address = self.site.address
+
+ if not self.hasSitePermission(address):
+ return self.response(to, {"error": "Forbidden"})
+
+ site = self.server.sites[address]
+
+ try:
+ del site.settings["optional_help"][directory]
+ self.response(to, "ok")
+ except Exception:
+ self.response(to, {"error": "Not found"})
+
+ def cbOptionalHelpAll(self, to, site, value):
+ site.settings["autodownloadoptional"] = value
+ self.response(to, value)
+
+ def actionOptionalHelpAll(self, to, value, address=None):
+ if not address:
+ address = self.site.address
+
+ if not self.hasSitePermission(address):
+ return self.response(to, {"error": "Forbidden"})
+
+ site = self.server.sites[address]
+
+ if value:
+ if "ADMIN" in self.site.settings["permissions"]:
+ self.cbOptionalHelpAll(to, site, True)
+ else:
+ site_title = site.content_manager.contents["content.json"].get("title", address)
+ self.cmd(
+ "confirm",
+ [
+ "Help distribute all new optional files on site %s" % cgi.escape(site_title),
+ "Yes, I want to help!"
+ ],
+ lambda (res): self.cbOptionalHelpAll(to, site, True)
+ )
+ else:
+ site.settings["autodownloadoptional"] = False
+ self.response(to, False)
diff --git a/plugins/OptionalManager/__init__.py b/plugins/OptionalManager/__init__.py
new file mode 100644
index 00000000..02969bba
--- /dev/null
+++ b/plugins/OptionalManager/__init__.py
@@ -0,0 +1 @@
+import OptionalManagerPlugin
\ No newline at end of file