diff --git a/plugins/ContentFilter/ContentFilterPlugin.py b/plugins/ContentFilter/ContentFilterPlugin.py
new file mode 100644
index 00000000..bde4ce01
--- /dev/null
+++ b/plugins/ContentFilter/ContentFilterPlugin.py
@@ -0,0 +1,202 @@
+import time
+import re
+
+from Plugin import PluginManager
+from Translate import Translate
+from Config import config
+
+from ContentFilterStorage import ContentFilterStorage
+
+
+if "_" not in locals():
+ _ = Translate("plugins/ContentFilter/languages/")
+
+
+@PluginManager.registerTo("SiteManager")
+class SiteManagerPlugin(object):
+ def load(self, *args, **kwargs):
+ global filter_storage
+ super(SiteManagerPlugin, self).load(*args, **kwargs)
+ filter_storage = ContentFilterStorage(site_manager=self)
+
+
+@PluginManager.registerTo("UiWebsocket")
+class UiWebsocketPlugin(object):
+ # Mute
+ def cbMuteAdd(self, to, auth_address, cert_user_id, reason):
+ filter_storage.file_content["mutes"][auth_address] = {
+ "cert_user_id": cert_user_id, "reason": reason, "source": self.site.address, "date_added": time.time()
+ }
+ filter_storage.save()
+ filter_storage.changeDbs(auth_address, "remove")
+ self.response(to, "ok")
+
+ def actionMuteAdd(self, to, auth_address, cert_user_id, reason):
+ if "ADMIN" in self.getPermissions(to):
+ self.cbMuteAdd(to, auth_address, cert_user_id, reason)
+ else:
+ self.cmd(
+ "confirm",
+ [_["Hide all content from %s?"] % cert_user_id, _["Mute"]],
+ lambda (res): self.cbMuteAdd(to, auth_address, cert_user_id, reason)
+ )
+
+ def cbMuteRemove(self, to, auth_address):
+ del filter_storage.file_content["mutes"][auth_address]
+ filter_storage.save()
+ filter_storage.changeDbs(auth_address, "load")
+ self.response(to, "ok")
+
+ def actionMuteRemove(self, to, auth_address):
+ if "ADMIN" in self.getPermissions(to):
+ self.cbMuteRemove(to, auth_address)
+ else:
+ self.cmd(
+ "confirm",
+ [_["Unmute %s?"] % filter_storage.file_content["mutes"][auth_address]["cert_user_id"], _["Unmute"]],
+ lambda (res): self.cbMuteRemove(to, auth_address)
+ )
+
+ def actionMuteList(self, to):
+ if "ADMIN" in self.getPermissions(to):
+ self.response(to, filter_storage.file_content["mutes"])
+ else:
+ return self.response(to, {"error": "Forbidden: Only ADMIN sites can list mutes"})
+
+ # Siteblock
+ def actionSiteblockAdd(self, to, site_address, reason=None):
+ if "ADMIN" not in self.getPermissions(to):
+ return self.response(to, {"error": "Forbidden: Only ADMIN sites can add to blocklist"})
+ filter_storage.file_content["siteblocks"][site_address] = {"date_added": time.time(), "reason": reason}
+ filter_storage.save()
+ self.response(to, "ok")
+
+ def actionSiteblockRemove(self, to, site_address):
+ if "ADMIN" not in self.getPermissions(to):
+ return self.response(to, {"error": "Forbidden: Only ADMIN sites can remove from blocklist"})
+ del filter_storage.file_content["siteblocks"][site_address]
+ filter_storage.save()
+ self.response(to, "ok")
+
+ def actionSiteblockList(self, to):
+ if "ADMIN" in self.getPermissions(to):
+ self.response(to, filter_storage.file_content["siteblocks"])
+ else:
+ return self.response(to, {"error": "Forbidden: Only ADMIN sites can list blocklists"})
+
+ # Include
+ def actionFilterIncludeAdd(self, to, inner_path, description=None, address=None):
+ if address:
+ if "ADMIN" not in self.getPermissions(to):
+ return self.response(to, {"error": "Forbidden: Only ADMIN sites can manage different site include"})
+ site = self.server.sites[address]
+ else:
+ address = self.site.address
+ site = self.site
+
+ if "ADMIN" in self.getPermissions(to):
+ self.cbFilterIncludeAdd(to, True, address, inner_path, description)
+ else:
+ content = site.storage.loadJson(inner_path)
+ title = _["New shared global content filter: %s (%s sites, %s users)"] % (
+ inner_path, len(content.get("siteblocks", {})), len(content.get("mutes", {}))
+ )
+
+ self.cmd(
+ "confirm",
+ [title, "Add"],
+ lambda (res): self.cbFilterIncludeAdd(to, res, address, inner_path, description)
+ )
+
+ def cbFilterIncludeAdd(self, to, res, address, inner_path, description):
+ if not res:
+ self.response(to, res)
+ return False
+
+ filter_storage.includeAdd(address, inner_path, description)
+ self.response(to, "ok")
+
+ def actionFilterIncludeRemove(self, to, inner_path, address=None):
+ if address:
+ if "ADMIN" not in self.getPermissions(to):
+ return self.response(to, {"error": "Forbidden: Only ADMIN sites can manage different site include"})
+ else:
+ address = self.site.address
+
+ key = "%s/%s" % (address, inner_path)
+ if key not in filter_storage.file_content["includes"]:
+ self.response(to, {"error": "Include not found"})
+ filter_storage.includeRemove(address, inner_path)
+ self.response(to, "ok")
+
+ def actionFilterIncludeList(self, to, all_sites=False, filters=False):
+ if all_sites and "ADMIN" not in self.getPermissions(to):
+ return self.response(to, {"error": "Forbidden: Only ADMIN sites can list all sites includes"})
+
+ back = []
+ includes = filter_storage.file_content.get("includes", {}).values()
+ for include in includes:
+ if not all_sites and include["address"] != self.site.address:
+ continue
+ if filters:
+ content = filter_storage.site_manager.get(include["address"]).storage.loadJson(include["inner_path"])
+ include["mutes"] = content.get("mutes", {})
+ include["siteblocks"] = content.get("siteblocks", {})
+ back.append(include)
+ self.response(to, back)
+
+
+@PluginManager.registerTo("SiteStorage")
+class SiteStoragePlugin(object):
+ def updateDbFile(self, inner_path, file=None, cur=None):
+ if file is not False: # File deletion always allowed
+ # Find for bitcoin addresses in file path
+ matches = re.findall("/(1[A-Za-z0-9]{26,35})/", inner_path)
+ # Check if any of the adresses are in the mute list
+ for auth_address in matches:
+ if filter_storage.isMuted(auth_address):
+ self.log.debug("Mute match: %s, ignoring %s" % (auth_address, inner_path))
+ return False
+
+ return super(SiteStoragePlugin, self).updateDbFile(inner_path, file=file, cur=cur)
+
+ def onUpdated(self, inner_path, file=None):
+ file_path = "%s/%s" % (self.site.address, inner_path)
+ if file_path in filter_storage.file_content["includes"]:
+ self.log.debug("Filter file updated: %s" % inner_path)
+ filter_storage.includeUpdateAll()
+ return super(SiteStoragePlugin, self).onUpdated(inner_path, file=file)
+
+
+@PluginManager.registerTo("UiRequest")
+class UiRequestPlugin(object):
+ def actionWrapper(self, path, extra_headers=None):
+ match = re.match("/(?P[A-Za-z0-9\._-]+)(?P/.*|$)", path)
+ if not match:
+ return False
+ address = match.group("address")
+
+ if self.server.site_manager.get(address): # Site already exists
+ return super(UiRequestPlugin, self).actionWrapper(path, extra_headers)
+
+ if self.server.site_manager.isDomain(address):
+ address = self.server.site_manager.resolveDomain(address)
+
+ if filter_storage.isSiteblocked(address):
+ site = self.server.site_manager.get(config.homepage)
+ if not extra_headers:
+ extra_headers = {}
+ self.sendHeader(extra_headers=extra_headers)
+ return iter([super(UiRequestPlugin, self).renderWrapper(
+ site, path, "uimedia/plugins/contentfilter/blocklisted.html?address=" + address,
+ "Blacklisted site", extra_headers, show_loadingscreen=False
+ )])
+ else:
+ return super(UiRequestPlugin, self).actionWrapper(path, extra_headers)
+
+ def actionUiMedia(self, path, *args, **kwargs):
+ if path.startswith("/uimedia/plugins/contentfilter/"):
+ file_path = path.replace("/uimedia/plugins/contentfilter/", "plugins/ContentFilter/media/")
+ return self.actionFile(file_path)
+ else:
+ return super(UiRequestPlugin, self).actionUiMedia(path)
diff --git a/plugins/ContentFilter/ContentFilterStorage.py b/plugins/ContentFilter/ContentFilterStorage.py
new file mode 100644
index 00000000..17af298f
--- /dev/null
+++ b/plugins/ContentFilter/ContentFilterStorage.py
@@ -0,0 +1,140 @@
+import os
+import json
+import logging
+import collections
+import time
+
+from Debug import Debug
+from Plugin import PluginManager
+from Config import config
+from util import helper
+
+class ContentFilterStorage(object):
+ def __init__(self, site_manager):
+ self.log = logging.getLogger("ContentFilterStorage")
+ self.file_path = "%s/filters.json" % config.data_dir
+ self.site_manager = site_manager
+ self.file_content = self.load()
+
+ # Set default values for filters.json
+ if not self.file_content:
+ self.file_content = {}
+
+ # Site blacklist renamed to site blocks
+ if "site_blacklist" in self.file_content:
+ self.file_content["siteblocks"] = self.file_content["site_blacklist"]
+ del self.file_content["site_blacklist"]
+
+ for key in ["mutes", "siteblocks", "includes"]:
+ if key not in self.file_content:
+ self.file_content[key] = {}
+
+ self.include_filters = collections.defaultdict(set) # Merged list of mutes and blacklists from all include
+ self.includeUpdateAll(update_site_dbs=False)
+
+ def load(self):
+ # Rename previously used mutes.json -> filters.json
+ if os.path.isfile("%s/mutes.json" % config.data_dir):
+ self.log.info("Renaming mutes.json to filters.json...")
+ os.rename("%s/mutes.json" % config.data_dir, self.file_path)
+ if os.path.isfile(self.file_path):
+ try:
+ return json.load(open(self.file_path))
+ except Exception as err:
+ self.log.error("Error loading filters.json: %s" % err)
+ return None
+ else:
+ return None
+
+ def includeUpdateAll(self, update_site_dbs=True):
+ s = time.time()
+ new_include_filters = collections.defaultdict(set)
+
+ # Load all include files data into a merged set
+ for include_path in self.file_content["includes"]:
+ address, inner_path = include_path.split("/", 1)
+ try:
+ content = self.site_manager.get(address).storage.loadJson(inner_path)
+ except Exception as err:
+ self.log.warning(
+ "Error loading include %s: %s" %
+ (include_path, Debug.formatException(err))
+ )
+ continue
+
+ for key, val in content.iteritems():
+ if type(val) is not dict:
+ continue
+
+ new_include_filters[key].update(val.keys())
+
+ mutes_added = new_include_filters["mutes"].difference(self.include_filters["mutes"])
+ mutes_removed = self.include_filters["mutes"].difference(new_include_filters["mutes"])
+
+ self.include_filters = new_include_filters
+
+ if update_site_dbs:
+ for auth_address in mutes_added:
+ self.changeDbs(auth_address, "remove")
+
+ for auth_address in mutes_removed:
+ if not self.isMuted(auth_address):
+ self.changeDbs(auth_address, "load")
+
+ num_mutes = len(self.include_filters["mutes"])
+ num_siteblocks = len(self.include_filters["siteblocks"])
+ self.log.debug(
+ "Loaded %s mutes, %s blocked sites from %s includes in %.3fs" %
+ (num_mutes, num_siteblocks, len(self.file_content["includes"]), time.time() - s)
+ )
+
+ def includeAdd(self, address, inner_path, description=None):
+ self.file_content["includes"]["%s/%s" % (address, inner_path)] = {
+ "date_added": time.time(),
+ "address": address,
+ "description": description,
+ "inner_path": inner_path
+ }
+ self.includeUpdateAll()
+ self.save()
+
+ def includeRemove(self, address, inner_path):
+ del self.file_content["includes"]["%s/%s" % (address, inner_path)]
+ self.includeUpdateAll()
+ self.save()
+
+ def save(self):
+ s = time.time()
+ helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True))
+ self.log.debug("Saved in %.3fs" % (time.time() - s))
+
+ def isMuted(self, auth_address):
+ if auth_address in self.file_content["mutes"] or auth_address in self.include_filters["mutes"]:
+ return True
+ else:
+ return False
+
+ def isSiteblocked(self, address):
+ if address in self.file_content["siteblocks"] or address in self.include_filters["siteblocks"]:
+ return True
+ else:
+ return False
+
+ # Search and remove or readd files of an user
+ def changeDbs(self, auth_address, action):
+ self.log.debug("Mute action %s on user %s" % (action, auth_address))
+ res = self.site_manager.list().values()[0].content_manager.contents.db.execute(
+ "SELECT * FROM content LEFT JOIN site USING (site_id) WHERE inner_path LIKE :inner_path",
+ {"inner_path": "%%/%s/%%" % auth_address}
+ )
+ for row in res:
+ site = self.site_manager.sites.get(row["address"])
+ if not site:
+ continue
+ dir_inner_path = helper.getDirname(row["inner_path"])
+ for file_name in site.storage.walk(dir_inner_path):
+ if action == "remove":
+ site.storage.onUpdated(dir_inner_path + file_name, False)
+ else:
+ site.storage.onUpdated(dir_inner_path + file_name)
+ site.onFileDone(dir_inner_path + file_name)
diff --git a/plugins/ContentFilter/Test/TestContentFilter.py b/plugins/ContentFilter/Test/TestContentFilter.py
new file mode 100644
index 00000000..e1b37b16
--- /dev/null
+++ b/plugins/ContentFilter/Test/TestContentFilter.py
@@ -0,0 +1,82 @@
+import pytest
+from ContentFilter import ContentFilterPlugin
+from Site import SiteManager
+
+
+@pytest.fixture
+def filter_storage():
+ ContentFilterPlugin.filter_storage = ContentFilterPlugin.ContentFilterStorage(SiteManager.site_manager)
+ return ContentFilterPlugin.filter_storage
+
+
+@pytest.mark.usefixtures("resetSettings")
+@pytest.mark.usefixtures("resetTempSettings")
+class TestContentFilter:
+ def createInclude(self, site):
+ site.storage.writeJson("filters.json", {
+ "mutes": {"1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C": {}},
+ "siteblocks": {site.address: {}}
+ })
+
+ def testIncludeLoad(self, site, filter_storage):
+ self.createInclude(site)
+ filter_storage.file_content["includes"]["%s/%s" % (site.address, "filters.json")] = {
+ "date_added": 1528295893,
+ }
+
+ assert not filter_storage.include_filters["mutes"]
+ assert not filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+ assert not filter_storage.isSiteblocked(site.address)
+ filter_storage.includeUpdateAll(update_site_dbs=False)
+ assert len(filter_storage.include_filters["mutes"]) == 1
+ assert filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+ assert filter_storage.isSiteblocked(site.address)
+
+ def testIncludeAdd(self, site, filter_storage):
+ self.createInclude(site)
+ query_num_json = "SELECT COUNT(*) AS num FROM json WHERE directory = 'users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C'"
+ assert not filter_storage.isSiteblocked(site.address)
+ assert not filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+ assert site.storage.query(query_num_json).fetchone()["num"] == 2
+
+ # Add include
+ filter_storage.includeAdd(site.address, "filters.json")
+
+ assert filter_storage.isSiteblocked(site.address)
+ assert filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+ assert site.storage.query(query_num_json).fetchone()["num"] == 0
+
+ # Remove include
+ filter_storage.includeRemove(site.address, "filters.json")
+
+ assert not filter_storage.isSiteblocked(site.address)
+ assert not filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+ assert site.storage.query(query_num_json).fetchone()["num"] == 2
+
+ def testIncludeChange(self, site, filter_storage):
+ self.createInclude(site)
+ filter_storage.includeAdd(site.address, "filters.json")
+ assert filter_storage.isSiteblocked(site.address)
+ assert filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
+
+ # Add new blocked site
+ assert not filter_storage.isSiteblocked("1Hello")
+
+ filter_content = site.storage.loadJson("filters.json")
+ filter_content["siteblocks"]["1Hello"] = {}
+ site.storage.writeJson("filters.json", filter_content)
+
+ assert filter_storage.isSiteblocked("1Hello")
+
+ # Add new muted user
+ query_num_json = "SELECT COUNT(*) AS num FROM json WHERE directory = 'users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q'"
+ assert not filter_storage.isMuted("1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q")
+ assert site.storage.query(query_num_json).fetchone()["num"] == 2
+
+ filter_content["mutes"]["1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q"] = {}
+ site.storage.writeJson("filters.json", filter_content)
+
+ assert filter_storage.isMuted("1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q")
+ assert site.storage.query(query_num_json).fetchone()["num"] == 0
+
+
diff --git a/plugins/ContentFilter/Test/conftest.py b/plugins/ContentFilter/Test/conftest.py
new file mode 100644
index 00000000..634e66e2
--- /dev/null
+++ b/plugins/ContentFilter/Test/conftest.py
@@ -0,0 +1 @@
+from src.Test.conftest import *
diff --git a/plugins/ContentFilter/Test/pytest.ini b/plugins/ContentFilter/Test/pytest.ini
new file mode 100644
index 00000000..d09210d1
--- /dev/null
+++ b/plugins/ContentFilter/Test/pytest.ini
@@ -0,0 +1,5 @@
+[pytest]
+python_files = Test*.py
+addopts = -rsxX -v --durations=6
+markers =
+ webtest: mark a test as a webtest.
\ No newline at end of file
diff --git a/plugins/ContentFilter/__init__.py b/plugins/ContentFilter/__init__.py
new file mode 100644
index 00000000..4d8c3acc
--- /dev/null
+++ b/plugins/ContentFilter/__init__.py
@@ -0,0 +1 @@
+import ContentFilterPlugin
diff --git a/plugins/ContentFilter/languages/hu.json b/plugins/ContentFilter/languages/hu.json
new file mode 100644
index 00000000..9b57e697
--- /dev/null
+++ b/plugins/ContentFilter/languages/hu.json
@@ -0,0 +1,6 @@
+{
+ "Hide all content from %s?": "%s tartalmaniak elrejtése?",
+ "Mute": "Elnémítás",
+ "Unmute %s?": "%s tartalmaniak megjelenítése?",
+ "Unmute": "Némítás visszavonása"
+}
diff --git a/plugins/ContentFilter/languages/it.json b/plugins/ContentFilter/languages/it.json
new file mode 100644
index 00000000..9a2c6761
--- /dev/null
+++ b/plugins/ContentFilter/languages/it.json
@@ -0,0 +1,6 @@
+{
+ "Hide all content from %s?": "%s Vuoi nascondere i contenuti di questo utente ?",
+ "Mute": "Attiva Silenzia",
+ "Unmute %s?": "%s Vuoi mostrare i contenuti di questo utente ?",
+ "Unmute": "Disattiva Silenzia"
+}
diff --git a/plugins/ContentFilter/languages/zh-tw.json b/plugins/ContentFilter/languages/zh-tw.json
new file mode 100644
index 00000000..0995f3a0
--- /dev/null
+++ b/plugins/ContentFilter/languages/zh-tw.json
@@ -0,0 +1,6 @@
+{
+ "Hide all content from %s?": "屏蔽 %s 的所有內容？",
+ "Mute": "屏蔽",
+ "Unmute %s?": "對 %s 解除屏蔽？",
+ "Unmute": "解除屏蔽"
+}
diff --git a/plugins/ContentFilter/languages/zh.json b/plugins/ContentFilter/languages/zh.json
new file mode 100644
index 00000000..bf63f107
--- /dev/null
+++ b/plugins/ContentFilter/languages/zh.json
@@ -0,0 +1,6 @@
+{
+ "Hide all content from %s?": "屏蔽 %s 的所有内容？",
+ "Mute": "屏蔽",
+ "Unmute %s?": "对 %s 解除屏蔽？",
+ "Unmute": "解除屏蔽"
+}
diff --git a/plugins/ContentFilter/media/blocklisted.html b/plugins/ContentFilter/media/blocklisted.html
new file mode 100644
index 00000000..33930b26
--- /dev/null
+++ b/plugins/ContentFilter/media/blocklisted.html
@@ -0,0 +1,86 @@
+
+
+
+
+
+
+
Site blocked
+
This site is on your blocklist:
+
+
Too much image
+
on 2015-01-25 12:32:11
+
+
+