From ee0829d95e4d030a5a5d5a3dd4699c10ec8a8026 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Fri, 22 Dec 2023 11:00:58 +0000 Subject: [PATCH 01/15] Minor CryptBitcoin refactor --- src/Crypt/CryptBitcoin.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/Crypt/CryptBitcoin.py b/src/Crypt/CryptBitcoin.py index a0807187..25377f5d 100644 --- a/src/Crypt/CryptBitcoin.py +++ b/src/Crypt/CryptBitcoin.py @@ -4,6 +4,9 @@ import binascii import time import hashlib +from collections.abc import Container +from typing import Optional + from util.Electrum import dbl_format from Config import config @@ -69,7 +72,8 @@ def privatekeyToAddress(privatekey): # Return address from private key return False -def sign(data, privatekey): # Return sign to data using private key +def sign(data: str, privatekey: str) -> str: + """Sign data with privatekey, return base64 string signature""" if privatekey.startswith("23") and len(privatekey) > 52: return None # Old style private key not supported return base64.b64encode(sslcurve.sign( @@ -79,13 +83,13 @@ def sign(data, privatekey): # Return sign to data using private key hash=dbl_format )).decode() - -def verify(data, valid_address, sign, lib_verify=None): # Verify data using address and sign +def get_sign_address_64(data: str, sign: str, lib_verify=None) -> Optional[str]: + """Returns pubkey/address of signer if any""" if not lib_verify: lib_verify = lib_verify_best if not sign: - return False + return None if lib_verify == "libsecp256k1": sign_address = libsecp256k1message.recover_address(data.encode("utf8"), sign).decode("utf8") @@ -95,10 +99,23 @@ def verify(data, valid_address, sign, lib_verify=None): # Verify data using add else: raise Exception("No library enabled for signature verification") - if type(valid_address) is list: # Any address in the list - return sign_address in valid_address - else: # One possible address - return sign_address == valid_address + return sign_address + +def verify(*args, **kwargs): + """Default verify, see verify64""" + return verify64(*args, **kwargs) + +def verify64(data: str, addresses: str | Container[str], sign: str, lib_verify=None) -> bool: + """Verify that sign is a valid signature for data by one of addresses + + Expecting signature to be in base64 + """ + sign_address = get_sign_address_64(data, sign, lib_verify) + + if isinstance(addresses, str): + return sign_address == addresses + else: + return sign_address in addresses def isValidAddress(addr): '''Check if provided address is valid bitcoin address''' From 31d94a16b62821931beeb5260c5d489a869c90f3 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Fri, 22 Dec 2023 12:59:04 +0000 Subject: [PATCH 02/15] Add option to ignore compromised certificates This would allow to stop spam attacks without pausing sites that allow compromised certificates and without action from site owners. However, without mass adoption of adding individual permissions on sites for valid users of such certificates, this will also block updates from such users. In any case this is more of a temporary measure in case we face such an attack before the whole user id issue is resolved --- src/Config.py | 2 ++ src/Content/ContentManager.py | 43 +++++++++++++++++++++++++++++++---- src/Ui/UiWebsocket.py | 3 +++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/Config.py b/src/Config.py index fee378d0..99619c47 100644 --- a/src/Config.py +++ b/src/Config.py @@ -297,6 +297,8 @@ class Config(object): self.parser.add_argument('--download-optional', choices=["manual", "auto"], default="manual") + self.parser.add_argument('--lax-cert-check', action=argparse.BooleanOptionalAction, default=True, help="Enabling lax cert check allows users getting site writing priviledges by employing compromized (i.e. with leaked private keys) cert issuer. Disable for spam protection") + self.parser.add_argument('--tor', help='enable: Use only for Tor peers, always: Use Tor for every connection', choices=["disable", "enable", "always"], default='enable') self.parser.add_argument('--tor-controller', help='Tor controller address', metavar='ip:port', default='127.0.0.1:9051') self.parser.add_argument('--tor-proxy', help='Tor proxy address', metavar='ip:port', default='127.0.0.1:9050') diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index a06ba523..938f3d6f 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -29,7 +29,17 @@ class SignError(Exception): @PluginManager.acceptPlugins -class ContentManager(object): +class ContentManager: + """Manage site content verifying and other related stuff""" + + def loadBadCerts(): + try: + with open(f'{config.data_dir}/badcerts.json') as f: + return set(json.load(f)) + except FileNotFoundError: + return set() + + bad_certs = loadBadCerts() def __init__(self, site): self.site = site @@ -38,6 +48,13 @@ class ContentManager(object): self.hashfield = PeerHashfield() self.has_optional_files = False + def addBadCert(self, sign): + addr = CryptBitcoin.get_sign_address_64('compromised', sign) + if addr: + self.bad_certs.add(addr) + with open(f'{config.data_dir}/badcerts.json', 'w') as f: + json.dump(list(self.bad_certs), f) + # Load all content.json files def loadContents(self): if len(self.contents) == 0: @@ -478,6 +495,9 @@ class ContentManager(object): return self.getUserContentRules(parent_content, inner_path, content) return False + def isGoodCert(self, cert): + return cert not in self.bad_certs + # Get rules for a user file # Return: The rules of the file or False if not allowed def getUserContentRules(self, parent_content, inner_path, content): @@ -511,7 +531,20 @@ class ContentManager(object): banned = False if "signers" in rules: rules["signers"] = rules["signers"][:] # Make copy of the signers - for permission_pattern, permission_rules in list(user_contents["permission_rules"].items()): # Regexp rules + + if content is not None: + name, domain = content['cert_user_id'].rsplit('@', 1) + cert_addresses = parent_content['user_contents']['cert_signers'].get(domain) + else: + cert_addresses = None + # to prevent spam, if cert is compromised, only allow personal rules + if config.lax_cert_check or cert_addresses is None or self.isGoodCert(cert_addresses[0]): + permission_rules = user_contents.get('permission_rules', {}).items() + if not self.isGoodCert(cert_addresses[0]): + self.log.warning('Accepting compromised certificate! This may lead to spam attack. Turn off with --no_lax_cert_check. Default behaviour may change in the future.') + else: + permission_rules = {} + for permission_pattern, permission_rules in permission_rules: # Regexp rules if not SafeRe.match(permission_pattern, user_urn): continue # Rule is not valid for user if permission_rules is None: @@ -892,9 +925,9 @@ class ContentManager(object): raise VerifyError("No rules") # Check include size limit - if rules.get("max_size") is not None: # Include size limit - if content_size > rules["max_size"]: - raise VerifyError("Include too large %sB > %sB" % (content_size, rules["max_size"])) + max_size = rules.get("max_size", 0) + if content_size > max_size: + raise VerifyError(f'Include too large {content_size}B > {max_size}B') if rules.get("max_size_optional") is not None: # Include optional files limit if content_size_optional > rules["max_size_optional"]: diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index e6f2f405..953c63d9 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -405,6 +405,9 @@ class UiWebsocket(object): def actionSiteBadFiles(self, to): return list(self.site.bad_files.keys()) + def actionBadCert(self, to, sign): + self.site.content_manager.addBadCert(sign) + # Join to an event channel def actionChannelJoin(self, to, channels): if type(channels) != list: From 0382aee68f1d5ce36761b8cc659dd7f8c8bca49c Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Tue, 26 Dec 2023 15:14:00 +0000 Subject: [PATCH 03/15] CHANGELOG --- CHANGELOG.md | 1 + src/Config.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 172f5f7b..edfcada8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - fix chromium compatibility (@caryoscelus) - better fix of local sites leak (@caryoscelus) - ipython-based repl via --repl for debug/interactive development (@caryoscelus) +- optional blocking of compromised id certificates for spam protection (@caryoscelus) - various improvements ### zeronet-conservancy 0.7.10 (2023-07-26) (18d35d3bed4f0683e99) diff --git a/src/Config.py b/src/Config.py index 99619c47..a950be09 100644 --- a/src/Config.py +++ b/src/Config.py @@ -15,7 +15,7 @@ class Config(object): self.version = "0.7.10+" self.user_agent = "conservancy" # DEPRECATED ; replace with git-generated commit - self.rev = 5140 + self.rev = 5141 self.user_agent_rev = 8192 self.argv = argv self.action = None From d17f4afdd039c7f2c34f9ce5bfe2efed22ad72a3 Mon Sep 17 00:00:00 2001 From: yggverse Date: Wed, 27 Mar 2024 20:33:52 +0200 Subject: [PATCH 04/15] fix IPv6 validation #263 --- src/Ui/UiRequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 34af96e2..a61b0439 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -82,7 +82,7 @@ class UiRequest: self.learnHost(host) return True - if ":" in host and helper.isIp(host.rsplit(":", 1)[0]): # Test without port + if ":" in host and helper.isIp(host.rsplit(":", 1)[0].lstrip("[").rstrip("]")): # Test without port self.learnHost(host) return True From 762c5f24029a1e6f034120065889dcbf22841c71 Mon Sep 17 00:00:00 2001 From: yggverse Date: Thu, 28 Mar 2024 15:52:19 +0200 Subject: [PATCH 05/15] remove square brackets from IPv6 host #263 #264 #267 --- src/Ui/UiRequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index a61b0439..4b11548f 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -544,7 +544,7 @@ class UiRequest: return server_url def getHostWithoutPort(self): - return ':'.join(self.env['HTTP_HOST'].split(':')[:-1]) + return ':'.join(self.env['HTTP_HOST'].split(':')[:-1]).lstrip("[").rstrip("]") def processQueryString(self, site, query_string): match = re.search("zeronet_peers=(.*?)(&|$)", query_string) From 658c685a451a95b83bf1df7988e5a21ed1541e63 Mon Sep 17 00:00:00 2001 From: yggverse Date: Thu, 28 Mar 2024 16:16:02 +0200 Subject: [PATCH 06/15] add configurable ipv6_testip #263 --- src/File/FileServer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index d1de4761..ee14d4ce 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -98,7 +98,10 @@ class FileServer(ConnectionServer): if config.tor == "always": return True # Test if we can connect to ipv6 address - ipv6_testip = "fcec:ae97:8902:d810:6c92:ec67:efb2:3ec5" + if config.ipv6_testip: + ipv6_testip = config.ipv6_testip + else: + ipv6_testip = "fcec:ae97:8902:d810:6c92:ec67:efb2:3ec5" try: sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) sock.connect((ipv6_testip, 80)) From 5858a68d1166ee1b1b783ad702d5e9f3dc4ba680 Mon Sep 17 00:00:00 2001 From: yggverse Date: Thu, 28 Mar 2024 16:22:44 +0200 Subject: [PATCH 07/15] add ipv6_testip to server info #263 --- src/Config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Config.py b/src/Config.py index 8a847bf3..8885f576 100644 --- a/src/Config.py +++ b/src/Config.py @@ -582,6 +582,7 @@ class Config: "fileserver_port": self.fileserver_port, "ui_ip": self.ui_ip, "ui_port": self.ui_port, + "ipv6_testip": self.ipv6_testip, "version": self.version, "rev": self.rev, "language": self.language, From 4030d3a8592219151cf86402ed02a074b22479b1 Mon Sep 17 00:00:00 2001 From: yggverse Date: Thu, 28 Mar 2024 16:28:58 +0200 Subject: [PATCH 08/15] remove ipv6_testip info as dependent #263 --- src/Config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Config.py b/src/Config.py index 8885f576..8a847bf3 100644 --- a/src/Config.py +++ b/src/Config.py @@ -582,7 +582,6 @@ class Config: "fileserver_port": self.fileserver_port, "ui_ip": self.ui_ip, "ui_port": self.ui_port, - "ipv6_testip": self.ipv6_testip, "version": self.version, "rev": self.rev, "language": self.language, From 8fb98991ef9febe38cc83f8b7ea3b7e7041d6da9 Mon Sep 17 00:00:00 2001 From: yggverse Date: Thu, 28 Mar 2024 16:31:44 +0200 Subject: [PATCH 09/15] fix optional ipv6_testip check #263 --- src/File/FileServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index ee14d4ce..212cd301 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -98,7 +98,7 @@ class FileServer(ConnectionServer): if config.tor == "always": return True # Test if we can connect to ipv6 address - if config.ipv6_testip: + if hasattr(myObject, "ipv6_testip"): ipv6_testip = config.ipv6_testip else: ipv6_testip = "fcec:ae97:8902:d810:6c92:ec67:efb2:3ec5" From 3c24e5d34c6b0101be089be3d5ccde8c142d949c Mon Sep 17 00:00:00 2001 From: yggverse Date: Thu, 28 Mar 2024 16:32:30 +0200 Subject: [PATCH 10/15] fix object name #263 --- src/File/FileServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 212cd301..c74a288d 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -98,7 +98,7 @@ class FileServer(ConnectionServer): if config.tor == "always": return True # Test if we can connect to ipv6 address - if hasattr(myObject, "ipv6_testip"): + if hasattr(config, "ipv6_testip"): ipv6_testip = config.ipv6_testip else: ipv6_testip = "fcec:ae97:8902:d810:6c92:ec67:efb2:3ec5" From b928cfc0d5ac8c846045d398075942adf17d63d2 Mon Sep 17 00:00:00 2001 From: yggverse Date: Thu, 28 Mar 2024 16:48:01 +0200 Subject: [PATCH 11/15] add ipv6 support to openBrowser method #263 --- src/util/helper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/util/helper.py b/src/util/helper.py index af65f727..8c7c6fff 100644 --- a/src/util/helper.py +++ b/src/util/helper.py @@ -325,7 +325,10 @@ def openBrowser(agent): if agent and agent != "False": print(f"Opening browser: {agent}...") ui_ip = config.ui_ip if config.ui_ip != "*" else "127.0.0.1" - url = f'http://{ui_ip}:{config.ui_port}/{config.homepage}' + if ':' in ui_ip: # IPv6 + url = f'http://[{ui_ip}]:{config.ui_port}/{config.homepage}' + else: # IPv4 + url = f'http://{ui_ip}:{config.ui_port}/{config.homepage}' try: import subprocess return subprocess.Popen([config.open_browser, url]) From f4e52fce5a279a89a8cdebad458c21cdb204df86 Mon Sep 17 00:00:00 2001 From: yggverse Date: Thu, 28 Mar 2024 17:06:57 +0200 Subject: [PATCH 12/15] rollback ipv6_testip feature as dependent of config implementation #263 #269 --- src/File/FileServer.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index c74a288d..d1de4761 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -98,10 +98,7 @@ class FileServer(ConnectionServer): if config.tor == "always": return True # Test if we can connect to ipv6 address - if hasattr(config, "ipv6_testip"): - ipv6_testip = config.ipv6_testip - else: - ipv6_testip = "fcec:ae97:8902:d810:6c92:ec67:efb2:3ec5" + ipv6_testip = "fcec:ae97:8902:d810:6c92:ec67:efb2:3ec5" try: sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) sock.connect((ipv6_testip, 80)) From 6702f331a3dbdfea6ba463dd370690a6f566c1df Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sat, 30 Mar 2024 14:31:22 +0000 Subject: [PATCH 13/15] IPv6/non-localhost IP fix - Content-Security-Policy: `frame-src *` for main UiRequest and `self` for user-content UiRequest - revert change in getHostWithoutPort to make ipv6 work --- src/Ui/UiRequest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 4b11548f..482096a3 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -382,10 +382,12 @@ class UiRequest: port = int(self.env['SERVER_PORT']) if port == config.ui_port: other_port = config.ui_site_port + frame_src = '*' else: other_port = config.ui_port - site_server = f'{host}:{other_port}' - headers["Content-Security-Policy"] = f"default-src 'none'; script-src 'nonce-{script_nonce}'; img-src 'self' blob: data:; style-src 'self' blob: 'unsafe-inline'; connect-src *; frame-src {site_server}" + frame_src = 'self' + + headers["Content-Security-Policy"] = f"default-src 'none'; script-src 'nonce-{script_nonce}'; img-src 'self' blob: data:; style-src 'self' blob: 'unsafe-inline'; connect-src *; frame-src {frame_src}" if allow_ajax: headers["Access-Control-Allow-Origin"] = "null" @@ -544,7 +546,7 @@ class UiRequest: return server_url def getHostWithoutPort(self): - return ':'.join(self.env['HTTP_HOST'].split(':')[:-1]).lstrip("[").rstrip("]") + return ':'.join(self.env['HTTP_HOST'].split(':')[:-1]) def processQueryString(self, site, query_string): match = re.search("zeronet_peers=(.*?)(&|$)", query_string) From e3b010175f519280adf8f42f01532cd9df5021bd Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 4 Apr 2024 23:40:32 +0000 Subject: [PATCH 14/15] --ui-ip-protect option to only apply recent privacy protection where it makes sense (i.e. on localhost) by default refs #263, #270 --- src/Config.py | 9 +++++++++ src/Ui/UiRequest.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Config.py b/src/Config.py index 8a847bf3..ba519a6b 100644 --- a/src/Config.py +++ b/src/Config.py @@ -241,6 +241,7 @@ class Config: self.parser.add_argument('--log-rotate-backup-count', help='Log rotate backup count', default=5, type=int) self.parser.add_argument('--language', help='Web interface language', default=language, metavar='language') + self.parser.add_argument('--ui-ip-protect', help="Protect UI server from being accessed through third-party pages and on unauthorized cross-origin pages (enabled by default when serving on localhost IPs; doesn't work with non-local IPs, need testing with host names)", choices=['always', 'local', 'off'], default='local') self.parser.add_argument('--ui-ip', help='Web interface bind address', default="127.0.0.1", metavar='ip') self.parser.add_argument('--ui-port', help='Web interface bind port', default=43110, type=int, metavar='port') self.parser.add_argument('--ui-site-port', help='Port for serving site content, defaults to ui_port+1', default=None, metavar='port') @@ -459,6 +460,14 @@ class Config: self.arguments = self.parser.parse_args(argv[1:]) if self.arguments.ui_site_port is None: self.arguments.ui_site_port = self.arguments.ui_port + 1 + if self.arguments.ui_ip_protect == 'always': + self.arguments.ui_check_cors = True + elif self.arguments.ui_ip_protect == 'off': + self.arguments.ui_check_cors = False + elif self.arguments.ui_ip_protect == 'local': + self.arguments.ui_check_cors = self.arguments.ui_ip == '127.0.0.1' or self.arguments.ui_ip == '::1' + else: + raise Exception("Wrong argparse result") def parseConfig(self, argv): argv = self.fixArgs(argv) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 482096a3..6e1d5e9e 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -148,7 +148,7 @@ class UiRequest: return False # Deny cross site requests - if not self.isSameOrigin(referer, url) and not self.hasCorsPermission(referer): + if not self.isSameOrigin(referer, url) or not self.hasCorsPermission(referer): return True return False @@ -172,7 +172,7 @@ class UiRequest: protocol = self.env['wsgi.url_scheme'] return self.actionRedirect(f'{protocol}://{host}:{config.ui_port}{path_info}?{query_string}') - if self.isCrossOriginRequest(): + if config.ui_check_cors and self.isCrossOriginRequest(): # we are still exposed by answering on port self.log.warning('Cross-origin request detected. Someone might be trying to analyze your 0net usage') return [] From 77720365590f5cfde50335c0c251d3a06012b8d5 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Fri, 5 Apr 2024 10:48:22 +0000 Subject: [PATCH 15/15] Fix some CORS/redirectering cases --- src/Ui/UiRequest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 6e1d5e9e..70470dba 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -148,7 +148,7 @@ class UiRequest: return False # Deny cross site requests - if not self.isSameOrigin(referer, url) or not self.hasCorsPermission(referer): + if not self.isSameOrigin(referer, url) and not self.hasCorsPermission(referer): return True return False @@ -165,7 +165,7 @@ class UiRequest: is_navigate = self.env.get('HTTP_SEC_FETCH_MODE') == 'navigate' is_iframe = self.env.get('HTTP_SEC_FETCH_DEST') == 'iframe' - if is_navigate and not is_iframe and self.is_data_request: + if ((is_navigate and not is_iframe) or not config.ui_check_cors) and self.is_data_request: host = self.getHostWithoutPort() path_info = self.env['PATH_INFO'] query_string = self.env['QUERY_STRING']