From 8700351f3315ad7bd2abb593f112b8c075899257 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 20 Jul 2023 16:16:54 +0000 Subject: [PATCH 01/46] remove unused broken code --- src/Content/ContentDbDict.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/Content/ContentDbDict.py b/src/Content/ContentDbDict.py index 01df0427..7e1bca05 100644 --- a/src/Content/ContentDbDict.py +++ b/src/Content/ContentDbDict.py @@ -95,17 +95,6 @@ class ContentDbDict(dict): back.append((key, val)) return back - def values(self): - back = [] - for key, val in dict.iteritems(self): - if not val: - try: - val = self.loadItem(key) - except Exception: - continue - back.append(val) - return back - def get(self, key, default=None): try: return self.__getitem__(key) From c09b5e16c9f4646b46cb6ed3ab8202217fc4613f Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 20 Jul 2023 16:29:04 +0000 Subject: [PATCH 02/46] sitePublish --recursive option --- src/Config.py | 2 ++ src/main.py | 70 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/src/Config.py b/src/Config.py index 9f5de779..8a101cde 100644 --- a/src/Config.py +++ b/src/Config.py @@ -309,6 +309,8 @@ class Config(object): default=15441, nargs='?') action.add_argument('--inner_path', help='Content.json you want to publish (default: content.json)', default="content.json", metavar="inner_path") + action.add_argument('--recursive', help="Whether to publish all of site's content.json. " + "Overrides --inner_path. (default: false)", action='store_true', dest='recursive') # SiteVerify action = self.subparsers.add_parser("siteVerify", help='Verify site files using sha512: address') diff --git a/src/main.py b/src/main.py index b4b656db..57735ae2 100644 --- a/src/main.py +++ b/src/main.py @@ -422,47 +422,61 @@ class Actions(object): ws = websocket.create_connection(ws_address) return ws - def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json"): - global file_server - from Site.Site import Site + def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json", recursive=False): from Site import SiteManager - from File import FileServer # We need fileserver to handle incoming file requests - from Peer import Peer - file_server = FileServer() - site = SiteManager.site_manager.get(address) logging.info("Loading site...") + site = SiteManager.site_manager.get(address) site.settings["serving"] = True # Serving the site even if its disabled + if not recursive: + inner_paths = [inner_path] + else: + inner_paths = list(site.content_manager.contents.keys()) + try: ws = self.getWebsocket(site) + + except Exception as err: + self.sitePublishFallback(site, peer_ip, peer_port, inner_paths, err) + + else: logging.info("Sending siteReload") self.siteCmd(address, "siteReload", inner_path) - logging.info("Sending sitePublish") - self.siteCmd(address, "sitePublish", {"inner_path": inner_path, "sign": False}) + for inner_path in inner_paths: + logging.info(f"Sending sitePublish for {inner_path}") + self.siteCmd(address, "sitePublish", {"inner_path": inner_path, "sign": False}) logging.info("Done.") + ws.close() - except Exception as err: - logging.info("Can't connect to local websocket client: %s" % err) - logging.info("Creating FileServer....") - file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity - time.sleep(0.001) + def sitePublishFallback(self, site, peer_ip, peer_port, inner_paths, err): + if err is not None: + logging.info(f"Can't connect to local websocket client: {err}") + logging.info("Publish using fallback mechanism. " + "Note that there might be not enough time for peer discovery, " + "but you can specify target peer on command line.") + logging.info("Creating FileServer....") + file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity + time.sleep(0.001) - # Started fileserver - file_server.portCheck() - if peer_ip: # Announce ip specificed - site.addPeer(peer_ip, peer_port) - else: # Just ask the tracker - logging.info("Gathering peers from tracker") - site.announce() # Gather peers + # Started fileserver + file_server.portCheck() + if peer_ip: # Announce ip specificed + site.addPeer(peer_ip, peer_port) + else: # Just ask the tracker + logging.info("Gathering peers from tracker") + site.announce() # Gather peers + + for inner_path in inner_paths: published = site.publish(5, inner_path) # Push to peers - if published > 0: - time.sleep(3) - logging.info("Serving files (max 60s)...") - gevent.joinall([file_server_thread], timeout=60) - logging.info("Done.") - else: - logging.info("No peers found, sitePublish command only works if you already have visitors serving your site") + + if published > 0: + time.sleep(3) + logging.info("Serving files (max 60s)...") + gevent.joinall([file_server_thread], timeout=60) + logging.info("Done.") + else: + logging.info("No peers found, sitePublish command only works if you already have visitors serving your site") # Crypto commands def cryptPrivatekeyToAddress(self, privatekey=None): From 260c4e626bc9021ae3e92d80b920ff9fa44c0fc2 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 20 Jul 2023 18:28:04 +0000 Subject: [PATCH 03/46] make tests launch --- src/Test/conftest.py | 3 --- src/Test/pytest.ini | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Test/conftest.py b/src/Test/conftest.py index c8739086..57c61f3a 100644 --- a/src/Test/conftest.py +++ b/src/Test/conftest.py @@ -63,9 +63,6 @@ config.debug = True os.chdir(os.path.abspath(os.path.dirname(__file__) + "/../..")) # Set working dir -all_loaded = PluginManager.plugin_manager.loadPlugins() -assert all_loaded, "Not all plugin loaded successfully" - config.loadPlugins() config.parse(parse_config=False) # Parse again to add plugin configuration options diff --git a/src/Test/pytest.ini b/src/Test/pytest.ini index 556389a2..0ffb385f 100644 --- a/src/Test/pytest.ini +++ b/src/Test/pytest.ini @@ -1,6 +1,6 @@ [pytest] python_files = Test*.py -addopts = -rsxX -v --durations=6 --no-print-logs --capture=fd +addopts = -rsxX -v --durations=6 --capture=fd markers = slow: mark a tests as slow. webtest: mark a test as a webtest. From 51a6eaa057f07deda864355fd2bc62d19072f1a0 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 20 Jul 2023 18:28:21 +0000 Subject: [PATCH 04/46] fix threadpool test --- src/Test/TestThreadPool.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/Test/TestThreadPool.py b/src/Test/TestThreadPool.py index 5e95005e..5e71155d 100644 --- a/src/Test/TestThreadPool.py +++ b/src/Test/TestThreadPool.py @@ -15,11 +15,9 @@ class TestThreadPool: @pool.wrap def blocker(): events.append("S") - out = 0 - for i in range(10000000): - if i == 3000000: - events.append("M") - out += 1 + time.sleep(0.001) + events.append("M") + time.sleep(0.001) events.append("D") return out @@ -30,9 +28,6 @@ class TestThreadPool: assert events == ["S"] * 3 + ["M"] * 3 + ["D"] * 3 - res = blocker() - assert res == 10000000 - def testLockBlockingSameThread(self): lock = ThreadPool.Lock() From f8e24d47e34cc234b4e37e369f0b84f2efee5217 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 20 Jul 2023 18:38:20 +0000 Subject: [PATCH 05/46] remove ZeronameLocal plugin we no longer need it as all domains have been freezed refs #23 --- .../SiteManagerPlugin.py | 180 ------------------ .../disabled-ZeronameLocal/UiRequestPlugin.py | 39 ---- plugins/disabled-ZeronameLocal/__init__.py | 2 - 3 files changed, 221 deletions(-) delete mode 100644 plugins/disabled-ZeronameLocal/SiteManagerPlugin.py delete mode 100644 plugins/disabled-ZeronameLocal/UiRequestPlugin.py delete mode 100644 plugins/disabled-ZeronameLocal/__init__.py diff --git a/plugins/disabled-ZeronameLocal/SiteManagerPlugin.py b/plugins/disabled-ZeronameLocal/SiteManagerPlugin.py deleted file mode 100644 index 579e31c1..00000000 --- a/plugins/disabled-ZeronameLocal/SiteManagerPlugin.py +++ /dev/null @@ -1,180 +0,0 @@ -import logging, json, os, re, sys, time, socket -from Plugin import PluginManager -from Config import config -from Debug import Debug -from http.client import HTTPSConnection, HTTPConnection, HTTPException -from base64 import b64encode - -allow_reload = False # No reload supported - -@PluginManager.registerTo("SiteManager") -class SiteManagerPlugin(object): - def load(self, *args, **kwargs): - super(SiteManagerPlugin, self).load(*args, **kwargs) - self.log = logging.getLogger("ZeronetLocal Plugin") - self.error_message = None - if not config.namecoin_host or not config.namecoin_rpcport or not config.namecoin_rpcuser or not config.namecoin_rpcpassword: - self.error_message = "Missing parameters" - self.log.error("Missing parameters to connect to namecoin node. Please check all the arguments needed with '--help'. Zeronet will continue working without it.") - return - - url = "%(host)s:%(port)s" % {"host": config.namecoin_host, "port": config.namecoin_rpcport} - self.c = HTTPConnection(url, timeout=3) - user_pass = "%(user)s:%(password)s" % {"user": config.namecoin_rpcuser, "password": config.namecoin_rpcpassword} - userAndPass = b64encode(bytes(user_pass, "utf-8")).decode("ascii") - self.headers = {"Authorization" : "Basic %s" % userAndPass, "Content-Type": " application/json " } - - payload = json.dumps({ - "jsonrpc": "2.0", - "id": "zeronet", - "method": "ping", - "params": [] - }) - - try: - self.c.request("POST", "/", payload, headers=self.headers) - response = self.c.getresponse() - data = response.read() - self.c.close() - if response.status == 200: - result = json.loads(data.decode())["result"] - else: - raise Exception(response.reason) - except Exception as err: - self.log.error("The Namecoin node is unreachable. Please check the configuration value are correct. Zeronet will continue working without it.") - self.error_message = err - self.cache = dict() - - # Checks if it's a valid address - def isAddress(self, address): - return self.isBitDomain(address) or super(SiteManagerPlugin, self).isAddress(address) - - # Return: True if the address is domain - def isDomain(self, address): - return self.isBitDomain(address) or super(SiteManagerPlugin, self).isDomain(address) - - # Return: True if the address is .bit domain - def isBitDomain(self, address): - return re.match(r"(.*?)([A-Za-z0-9_-]+\.bit)$", address) - - # Return: Site object or None if not found - def get(self, address): - if self.isBitDomain(address): # Its looks like a domain - address_resolved = self.resolveDomain(address) - if address_resolved: # Domain found - site = self.sites.get(address_resolved) - if site: - site_domain = site.settings.get("domain") - if site_domain != address: - site.settings["domain"] = address - else: # Domain not found - site = self.sites.get(address) - - else: # Access by site address - site = super(SiteManagerPlugin, self).get(address) - return site - - # Return or create site and start download site files - # Return: Site or None if dns resolve failed - def need(self, address, *args, **kwargs): - if self.isBitDomain(address): # Its looks like a domain - address_resolved = self.resolveDomain(address) - if address_resolved: - address = address_resolved - else: - return None - - return super(SiteManagerPlugin, self).need(address, *args, **kwargs) - - # Resolve domain - # Return: The address or None - def resolveDomain(self, domain): - domain = domain.lower() - - #remove .bit on end - if domain[-4:] == ".bit": - domain = domain[0:-4] - - domain_array = domain.split(".") - - if self.error_message: - self.log.error("Not able to connect to Namecoin node : {!s}".format(self.error_message)) - return None - - if len(domain_array) > 2: - self.log.error("Too many subdomains! Can only handle one level (eg. staging.mixtape.bit)") - return None - - subdomain = "" - if len(domain_array) == 1: - domain = domain_array[0] - else: - subdomain = domain_array[0] - domain = domain_array[1] - - if domain in self.cache: - delta = time.time() - self.cache[domain]["time"] - if delta < 3600: - # Must have been less than 1hour - return self.cache[domain]["addresses_resolved"][subdomain] - - payload = json.dumps({ - "jsonrpc": "2.0", - "id": "zeronet", - "method": "name_show", - "params": ["d/"+domain] - }) - - try: - self.c.request("POST", "/", payload, headers=self.headers) - response = self.c.getresponse() - data = response.read() - self.c.close() - domain_object = json.loads(data.decode())["result"] - except Exception as err: - #domain doesn't exist - return None - - if "zeronet" in domain_object["value"]: - zeronet_domains = json.loads(domain_object["value"])["zeronet"] - - if isinstance(zeronet_domains, str): - # { - # "zeronet":"19rXKeKptSdQ9qt7omwN82smehzTuuq6S9" - # } is valid - zeronet_domains = {"": zeronet_domains} - - self.cache[domain] = {"addresses_resolved": zeronet_domains, "time": time.time()} - - elif "map" in domain_object["value"]: - # Namecoin standard use {"map": { "blog": {"zeronet": "1D..."} }} - data_map = json.loads(domain_object["value"])["map"] - - zeronet_domains = dict() - for subdomain in data_map: - if "zeronet" in data_map[subdomain]: - zeronet_domains[subdomain] = data_map[subdomain]["zeronet"] - if "zeronet" in data_map and isinstance(data_map["zeronet"], str): - # {"map":{ - # "zeronet":"19rXKeKptSdQ9qt7omwN82smehzTuuq6S9", - # }} - zeronet_domains[""] = data_map["zeronet"] - - self.cache[domain] = {"addresses_resolved": zeronet_domains, "time": time.time()} - - else: - # No Zeronet address registered - return None - - return self.cache[domain]["addresses_resolved"][subdomain] - -@PluginManager.registerTo("ConfigPlugin") -class ConfigPlugin(object): - def createArguments(self): - group = self.parser.add_argument_group("Zeroname Local plugin") - group.add_argument('--namecoin_host', help="Host to namecoin node (eg. 127.0.0.1)") - group.add_argument('--namecoin_rpcport', help="Port to connect (eg. 8336)") - group.add_argument('--namecoin_rpcuser', help="RPC user to connect to the namecoin node (eg. nofish)") - group.add_argument('--namecoin_rpcpassword', help="RPC password to connect to namecoin node") - - return super(ConfigPlugin, self).createArguments() diff --git a/plugins/disabled-ZeronameLocal/UiRequestPlugin.py b/plugins/disabled-ZeronameLocal/UiRequestPlugin.py deleted file mode 100644 index 0ccfb530..00000000 --- a/plugins/disabled-ZeronameLocal/UiRequestPlugin.py +++ /dev/null @@ -1,39 +0,0 @@ -import re -from Plugin import PluginManager - -@PluginManager.registerTo("UiRequest") -class UiRequestPlugin(object): - def __init__(self, *args, **kwargs): - from Site import SiteManager - self.site_manager = SiteManager.site_manager - super(UiRequestPlugin, self).__init__(*args, **kwargs) - - - # Media request - def actionSiteMedia(self, path): - match = re.match(r"/media/(?P
[A-Za-z0-9-]+\.[A-Za-z0-9\.-]+)(?P/.*|$)", path) - if match: # Its a valid domain, resolve first - domain = match.group("address") - address = self.site_manager.resolveDomain(domain) - if address: - path = "/media/"+address+match.group("inner_path") - return super(UiRequestPlugin, self).actionSiteMedia(path) # Get the wrapper frame output - - - # Is mediarequest allowed from that referer - def isMediaRequestAllowed(self, site_address, referer): - referer_path = re.sub("http[s]{0,1}://.*?/", "/", referer).replace("/media", "") # Remove site address - referer_path = re.sub(r"\?.*", "", referer_path) # Remove http params - - if self.isProxyRequest(): # Match to site domain - referer = re.sub("^http://zero[/]+", "http://", referer) # Allow /zero access - referer_site_address = re.match("http[s]{0,1}://(.*?)(/|$)", referer).group(1) - else: # Match to request path - referer_site_address = re.match(r"/(?P
[A-Za-z0-9\.-]+)(?P/.*|$)", referer_path).group("address") - - if referer_site_address == site_address: # Referer site address as simple address - return True - elif self.site_manager.resolveDomain(referer_site_address) == site_address: # Referer site address as dns - return True - else: # Invalid referer - return False diff --git a/plugins/disabled-ZeronameLocal/__init__.py b/plugins/disabled-ZeronameLocal/__init__.py deleted file mode 100644 index cf724069..00000000 --- a/plugins/disabled-ZeronameLocal/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from . import UiRequestPlugin -from . import SiteManagerPlugin \ No newline at end of file From c07334f2d518bd23b83bd77693061ac7ce5bd1de Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 20 Jul 2023 18:41:47 +0000 Subject: [PATCH 06/46] remove Dnschain plugin (alternative .bit resolution) as we freezed domains, this is no longer needed refs #23 --- .../disabled-Dnschain/SiteManagerPlugin.py | 153 ------------------ plugins/disabled-Dnschain/UiRequestPlugin.py | 34 ---- plugins/disabled-Dnschain/__init__.py | 3 - 3 files changed, 190 deletions(-) delete mode 100644 plugins/disabled-Dnschain/SiteManagerPlugin.py delete mode 100644 plugins/disabled-Dnschain/UiRequestPlugin.py delete mode 100644 plugins/disabled-Dnschain/__init__.py diff --git a/plugins/disabled-Dnschain/SiteManagerPlugin.py b/plugins/disabled-Dnschain/SiteManagerPlugin.py deleted file mode 100644 index 8b9508f1..00000000 --- a/plugins/disabled-Dnschain/SiteManagerPlugin.py +++ /dev/null @@ -1,153 +0,0 @@ -import logging, json, os, re, sys, time -import gevent -from Plugin import PluginManager -from Config import config -from util import Http -from Debug import Debug - -allow_reload = False # No reload supported - -log = logging.getLogger("DnschainPlugin") - -@PluginManager.registerTo("SiteManager") -class SiteManagerPlugin(object): - dns_cache_path = "%s/dns_cache.json" % config.data_dir - dns_cache = None - - # Checks if its a valid address - def isAddress(self, address): - if self.isDomain(address): - return True - else: - return super(SiteManagerPlugin, self).isAddress(address) - - - # Return: True if the address is domain - def isDomain(self, address): - return re.match(r"(.*?)([A-Za-z0-9_-]+\.[A-Za-z0-9]+)$", address) - - - # Load dns entries from data/dns_cache.json - def loadDnsCache(self): - if os.path.isfile(self.dns_cache_path): - self.dns_cache = json.load(open(self.dns_cache_path)) - else: - self.dns_cache = {} - log.debug("Loaded dns cache, entries: %s" % len(self.dns_cache)) - - - # Save dns entries to data/dns_cache.json - def saveDnsCache(self): - json.dump(self.dns_cache, open(self.dns_cache_path, "wb"), indent=2) - - - # Resolve domain using dnschain.net - # Return: The address or None - def resolveDomainDnschainNet(self, domain): - try: - match = self.isDomain(domain) - sub_domain = match.group(1).strip(".") - top_domain = match.group(2) - if not sub_domain: sub_domain = "@" - address = None - with gevent.Timeout(5, Exception("Timeout: 5s")): - res = Http.get("https://api.dnschain.net/v1/namecoin/key/%s" % top_domain).read() - data = json.loads(res)["data"]["value"] - if "zeronet" in data: - for key, val in data["zeronet"].items(): - self.dns_cache[key+"."+top_domain] = [val, time.time()+60*60*5] # Cache for 5 hours - self.saveDnsCache() - return data["zeronet"].get(sub_domain) - # Not found - return address - except Exception as err: - log.debug("Dnschain.net %s resolve error: %s" % (domain, Debug.formatException(err))) - - - # Resolve domain using dnschain.info - # Return: The address or None - def resolveDomainDnschainInfo(self, domain): - try: - match = self.isDomain(domain) - sub_domain = match.group(1).strip(".") - top_domain = match.group(2) - if not sub_domain: sub_domain = "@" - address = None - with gevent.Timeout(5, Exception("Timeout: 5s")): - res = Http.get("https://dnschain.info/bit/d/%s" % re.sub(r"\.bit$", "", top_domain)).read() - data = json.loads(res)["value"] - for key, val in data["zeronet"].items(): - self.dns_cache[key+"."+top_domain] = [val, time.time()+60*60*5] # Cache for 5 hours - self.saveDnsCache() - return data["zeronet"].get(sub_domain) - # Not found - return address - except Exception as err: - log.debug("Dnschain.info %s resolve error: %s" % (domain, Debug.formatException(err))) - - - # Resolve domain - # Return: The address or None - def resolveDomain(self, domain): - domain = domain.lower() - if self.dns_cache == None: - self.loadDnsCache() - if domain.count(".") < 2: # Its a topleved request, prepend @. to it - domain = "@."+domain - - domain_details = self.dns_cache.get(domain) - if domain_details and time.time() < domain_details[1]: # Found in cache and its not expired - return domain_details[0] - else: - # Resovle dns using dnschain - thread_dnschain_info = gevent.spawn(self.resolveDomainDnschainInfo, domain) - thread_dnschain_net = gevent.spawn(self.resolveDomainDnschainNet, domain) - gevent.joinall([thread_dnschain_net, thread_dnschain_info]) # Wait for finish - - if thread_dnschain_info.value and thread_dnschain_net.value: # Booth successfull - if thread_dnschain_info.value == thread_dnschain_net.value: # Same returned value - return thread_dnschain_info.value - else: - log.error("Dns %s missmatch: %s != %s" % (domain, thread_dnschain_info.value, thread_dnschain_net.value)) - - # Problem during resolve - if domain_details: # Resolve failed, but we have it in the cache - domain_details[1] = time.time()+60*60 # Dont try again for 1 hour - return domain_details[0] - else: # Not found in cache - self.dns_cache[domain] = [None, time.time()+60] # Don't check again for 1 min - return None - - - # Return or create site and start download site files - # Return: Site or None if dns resolve failed - def need(self, address, all_file=True): - if self.isDomain(address): # Its looks like a domain - address_resolved = self.resolveDomain(address) - if address_resolved: - address = address_resolved - else: - return None - - return super(SiteManagerPlugin, self).need(address, all_file) - - - # Return: Site object or None if not found - def get(self, address): - if self.sites == None: # Not loaded yet - self.load() - if self.isDomain(address): # Its looks like a domain - address_resolved = self.resolveDomain(address) - if address_resolved: # Domain found - site = self.sites.get(address_resolved) - if site: - site_domain = site.settings.get("domain") - if site_domain != address: - site.settings["domain"] = address - else: # Domain not found - site = self.sites.get(address) - - else: # Access by site address - site = self.sites.get(address) - return site - diff --git a/plugins/disabled-Dnschain/UiRequestPlugin.py b/plugins/disabled-Dnschain/UiRequestPlugin.py deleted file mode 100644 index 8ab9d5c5..00000000 --- a/plugins/disabled-Dnschain/UiRequestPlugin.py +++ /dev/null @@ -1,34 +0,0 @@ -import re -from Plugin import PluginManager - -@PluginManager.registerTo("UiRequest") -class UiRequestPlugin(object): - def __init__(self, server = None): - from Site import SiteManager - self.site_manager = SiteManager.site_manager - super(UiRequestPlugin, self).__init__(server) - - - # Media request - def actionSiteMedia(self, path): - match = re.match(r"/media/(?P
[A-Za-z0-9-]+\.[A-Za-z0-9\.-]+)(?P/.*|$)", path) - if match: # Its a valid domain, resolve first - domain = match.group("address") - address = self.site_manager.resolveDomain(domain) - if address: - path = "/media/"+address+match.group("inner_path") - return super(UiRequestPlugin, self).actionSiteMedia(path) # Get the wrapper frame output - - - # Is mediarequest allowed from that referer - def isMediaRequestAllowed(self, site_address, referer): - referer_path = re.sub("http[s]{0,1}://.*?/", "/", referer).replace("/media", "") # Remove site address - referer_site_address = re.match(r"/(?P
[A-Za-z0-9\.-]+)(?P/.*|$)", referer_path).group("address") - - if referer_site_address == site_address: # Referer site address as simple address - return True - elif self.site_manager.resolveDomain(referer_site_address) == site_address: # Referer site address as dns - return True - else: # Invalid referer - return False - diff --git a/plugins/disabled-Dnschain/__init__.py b/plugins/disabled-Dnschain/__init__.py deleted file mode 100644 index 2b36af5d..00000000 --- a/plugins/disabled-Dnschain/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# This plugin is experimental, if you really want to enable uncomment the following lines: -# import DnschainPlugin -# import SiteManagerPlugin \ No newline at end of file From 3330b19e3107945a8b0ef871a38d3d7dfeaa6cbc Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Fri, 21 Jul 2023 12:00:23 +0000 Subject: [PATCH 07/46] don't fail if http_accept header is */* fixes #67 --- src/Ui/UiRequest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index d30ff4e3..281a5e5c 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -398,8 +398,9 @@ class UiRequest(object): if self.isWebSocketRequest(): return self.error403("WebSocket request not allowed to load wrapper") # No websocket - if "text/html" not in self.env.get("HTTP_ACCEPT", ""): - return self.error403("Invalid Accept header to load wrapper: %s" % self.env.get("HTTP_ACCEPT", "")) + http_accept = self.env.get("HTTP_ACCEPT", "") + if "text/html" not in http_accept and "*/*" not in http_accept: + return self.error403(f"Invalid Accept header to load wrapper: {http_accept}") if "prefetch" in self.env.get("HTTP_X_MOZ", "") or "prefetch" in self.env.get("HTTP_PURPOSE", ""): return self.error403("Prefetch not allowed to load wrapper") From f94765f7b989263605c9148e1036e176fa71a45e Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sun, 23 Jul 2023 09:37:52 +0000 Subject: [PATCH 08/46] more anonymous upnp thanks to 0netdwf for reporting --- src/Peer/PeerPortchecker.py | 5 +---- src/util/UpnpPunch.py | 17 ++++++----------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/Peer/PeerPortchecker.py b/src/Peer/PeerPortchecker.py index f7ca68e3..8bf337cf 100644 --- a/src/Peer/PeerPortchecker.py +++ b/src/Peer/PeerPortchecker.py @@ -28,11 +28,8 @@ class PeerPortchecker(object): return urllib.request.urlopen(req, timeout=20.0) def portOpen(self, port): - # self.log.info("Not trying to open port using UpnpPunch until it's proven robust...") - # return False - try: - UpnpPunch.ask_to_open_port(port, 'ZeroNet', retries=3, protos=["TCP"]) + UpnpPunch.ask_to_open_port(port, retries=3, protos=["TCP"]) self.upnp_port_opened = True except Exception as err: self.log.warning("UpnpPunch run error: %s" % Debug.formatException(err)) diff --git a/src/util/UpnpPunch.py b/src/util/UpnpPunch.py index 2fa85209..4fd17c61 100644 --- a/src/util/UpnpPunch.py +++ b/src/util/UpnpPunch.py @@ -181,7 +181,6 @@ def _get_local_ips(): def _create_open_message(local_ip, port, - description="UPnPPunch", protocol="TCP", upnp_schema='WANIPConnection'): """ @@ -205,14 +204,13 @@ def _create_open_message(local_ip, """.format(port=port, protocol=protocol, host_ip=local_ip, - description=description, + description='', upnp_schema=upnp_schema) return (REMOVE_WHITESPACE.sub('><', soap_message), 'AddPortMapping') def _create_close_message(local_ip, port, - description=None, protocol='TCP', upnp_schema='WANIPConnection'): soap_message = """ @@ -294,12 +292,12 @@ def _send_requests(messages, location, upnp_schema, control_path): raise UpnpError('Sending requests using UPnP failed.') -def _orchestrate_soap_request(ip, port, msg_fn, desc=None, protos=("TCP", "UDP")): +def _orchestrate_soap_request(ip, port, msg_fn, protos=("TCP", "UDP")): logger.debug("Trying using local ip: %s" % ip) idg_data = _collect_idg_data(ip) soap_messages = [ - msg_fn(ip, port, desc, proto, idg_data['upnp_schema']) + msg_fn(ip, port, proto, idg_data['upnp_schema']) for proto in protos ] @@ -307,7 +305,6 @@ def _orchestrate_soap_request(ip, port, msg_fn, desc=None, protos=("TCP", "UDP") def _communicate_with_igd(port=15441, - desc="UpnpPunch", retries=3, fn=_create_open_message, protos=("TCP", "UDP")): @@ -321,7 +318,7 @@ def _communicate_with_igd(port=15441, def job(local_ip): for retry in range(retries): try: - _orchestrate_soap_request(local_ip, port, fn, desc, protos) + _orchestrate_soap_request(local_ip, port, fn, protos) return True except Exception as e: logger.debug('Upnp request using "{0}" failed: {1}'.format(local_ip, e)) @@ -357,20 +354,18 @@ def _communicate_with_igd(port=15441, return success -def ask_to_open_port(port=15441, desc="UpnpPunch", retries=3, protos=("TCP", "UDP")): +def ask_to_open_port(port=15441, retries=3, protos=("TCP", "UDP")): logger.debug("Trying to open port %d." % port) return _communicate_with_igd(port=port, - desc=desc, retries=retries, fn=_create_open_message, protos=protos) -def ask_to_close_port(port=15441, desc="UpnpPunch", retries=3, protos=("TCP", "UDP")): +def ask_to_close_port(port=15441, retries=3, protos=("TCP", "UDP")): logger.debug("Trying to close port %d." % port) # retries=1 because multiple successes cause 500 response and failure return _communicate_with_igd(port=port, - desc=desc, retries=retries, fn=_create_close_message, protos=protos) From cbf9c78e8ce4c3d0b27ad3d47c420797f93d0f3e Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sun, 23 Jul 2023 09:31:46 +0000 Subject: [PATCH 09/46] helper function: quickly check if a string looks like a valid Bitcoin address --- src/Crypt/CryptBitcoin.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Crypt/CryptBitcoin.py b/src/Crypt/CryptBitcoin.py index 68b2caa2..a0807187 100644 --- a/src/Crypt/CryptBitcoin.py +++ b/src/Crypt/CryptBitcoin.py @@ -99,3 +99,16 @@ def verify(data, valid_address, sign, lib_verify=None): # Verify data using add return sign_address in valid_address else: # One possible address return sign_address == valid_address + +def isValidAddress(addr): + '''Check if provided address is valid bitcoin address''' + if addr[0] != '1': + # no support for new-style addrs + return False + from base58 import b58decode + bs = b58decode(addr) + main = bs[:-4] + checksum = bs[-4:] + h1 = hashlib.sha256(main).digest() + h2 = hashlib.sha256(h1).digest() + return h2[:4] == checksum From 9d8be57025f0cf70795a62016f8fe385d4592382 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sun, 23 Jul 2023 18:07:45 +0000 Subject: [PATCH 10/46] bootstrap.url --- bootstrap.url | 1 + 1 file changed, 1 insertion(+) create mode 100644 bootstrap.url diff --git a/bootstrap.url b/bootstrap.url new file mode 100644 index 00000000..f23462eb --- /dev/null +++ b/bootstrap.url @@ -0,0 +1 @@ +https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.9/data-default-2023-07-23.zip \ No newline at end of file From 76d1b14eaf222cb084aa57e4b0105ed205482dc9 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sun, 23 Jul 2023 19:55:59 +0000 Subject: [PATCH 11/46] Ignore missing sites.json gracefully --- src/Site/Site.py | 6 +++++- src/Site/SiteManager.py | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index ffdb2bb0..ae2be36a 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -87,7 +87,11 @@ class Site(object): # Load site settings from data/sites.json def loadSettings(self, settings=None): if not settings: - settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address) + try: + settings = json.load(open(f'{config.data_dir}/sites.json')).get(self.address) + except Exception as err: + logging.error(f'Error loading {config.data_dir}/sites.json: {err}') + settings = {} if settings: self.settings = settings if "cache" not in settings: diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index d7ba6e94..78c20f86 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -31,17 +31,18 @@ class SiteManager(object): @util.Noparallel() def load(self, cleanup=True, startup=False): from .Site import Site - self.log.info("Loading sites... (cleanup: %s, startup: %s)" % (cleanup, startup)) + self.log.info(f'Loading sites... ({cleanup=}, {startup=})') self.loaded = False address_found = [] added = 0 load_s = time.time() # Load new adresses try: - json_path = "%s/sites.json" % config.data_dir + json_path = f"{config.data_dir}/sites.json" data = json.load(open(json_path)) except Exception as err: - raise Exception("Unable to load %s: %s" % (json_path, err)) + self.log.error(f"Unable to load {json_path}: {err}") + data = {} sites_need = [] From 5a184a5489b1da7f7c3c0fc88ab51e4c420d5cec Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sun, 23 Jul 2023 20:57:09 +0000 Subject: [PATCH 12/46] Don't hide warnings in console WARNING logging level is higher than INFO so really shouldn't hide them.. --- src/Config.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Config.py b/src/Config.py index 9f5de779..e1c397e2 100644 --- a/src/Config.py +++ b/src/Config.py @@ -797,10 +797,6 @@ class Config(object): except Exception as err: print("Can't change permission of %s: %s" % (self.log_dir, err)) - # Make warning hidden from console - logging.WARNING = 15 # Don't display warnings if not in debug mode - logging.addLevelName(15, "WARNING") - logging.getLogger('').name = "-" # Remove root prefix self.error_logger = ErrorLogHandler() From 9444e097a66f9ab12562a27231fa0ba4420f37ae Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sun, 23 Jul 2023 21:14:54 +0000 Subject: [PATCH 13/46] New command line action: importBundle; load bootstrap bundle on first run - importBundle imports zip archive of sites - latest bootstrapping bundle is downloaded if data/ dir is empty; primary reason for this for now is to avoid constantly updating tracker list in git tree and use Syncronite instead --- requirements.txt | 1 + src/Config.py | 6 ++++ src/main.py | 71 +++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4298ed61..887138d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ maxminddb rich defusedxml>=0.7 pyaes +requests diff --git a/src/Config.py b/src/Config.py index e1c397e2..8850679b 100644 --- a/src/Config.py +++ b/src/Config.py @@ -320,6 +320,10 @@ class Config(object): action.add_argument('cmd', help='API command name') action.add_argument('parameters', help='Parameters of the command', nargs='?') + # Import bundled sites + action = self.subparsers.add_parser("importBundle", help='Import sites from a .zip bundle') + action.add_argument('bundle', help='Path to a data bundle') + # dbRebuild action = self.subparsers.add_parser("dbRebuild", help='Rebuild site database cache') action.add_argument('address', help='Site to rebuild') @@ -426,6 +430,8 @@ class Config(object): self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true') self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port') self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip') + self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/caryoscelus/zeronet-conservancy/bootstrap/bootstrap.url', type=str) + self.parser.add_argument('--disable_bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true') self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*') self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', metavar='path', nargs='*') self.parser.add_argument('--trackers_proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable") diff --git a/src/main.py b/src/main.py index b4b656db..b99e0076 100644 --- a/src/main.py +++ b/src/main.py @@ -34,24 +34,72 @@ def load_config(): load_config() -def init_dirs(): - if not os.path.isdir(config.data_dir): - os.mkdir(config.data_dir) - try: - os.chmod(config.data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - except Exception as err: - startupError("Can't change permission of %s: %s" % (config.data_dir, err)) +def importBundle(bundle): + from zipfile import ZipFile + from Crypt.CryptBitcoin import isValidAddress + from Site import SiteManager + SiteManager.site_manager.load() - sites_json = f"{config.data_dir}/sites.json" + with ZipFile(bundle) as zf: + all_files = zf.namelist() + top_files = list(set(map(lambda f: f.split('/')[0], all_files))) + if len(top_files) == 1 and not isValidAddress(top_files[0]): + prefix = top_files[0]+'/' + else: + prefix = '' + top_2 = list(set(filter(lambda f: len(f)>0, + map(lambda f: f.removeprefix(prefix).split('/')[0], all_files)))) + for d in top_2: + if isValidAddress(d): + logging.info(f'unpack {d} into {config.data_dir}') + for fname in filter(lambda f: f.startswith(prefix+d) and not f.endswith('/'), all_files): + tgt = config.data_dir + '/' + fname.removeprefix(prefix) + logging.info(f'-- {fname} --> {tgt}') + info = zf.getinfo(fname) + info.filename = tgt + zf.extract(info) + logging.info(f'add site {d}') + SiteManager.site_manager.add(d, noload=True) + else: + logging.info(f'Warning: unknown file in a bundle: {prefix+d}') + SiteManager.site_manager.save() + +def init_dirs(): + data_dir = config.data_dir + if not os.path.isdir(data_dir): + os.mkdir(data_dir) + try: + os.chmod(data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + except Exception as err: + startupError(f"Can't change permission of {data_dir}: {err}") + + # download latest bootstrap bundle + if not config.disable_bootstrap and not config.offline: + import requests + from io import BytesIO + + print(f'fetching {config.bootstrap_url}') + response = requests.get(config.bootstrap_url) + if response.status_code != 200: + startupError(f"Cannot load bootstrap bundle (response status: {response.status_code})") + url = response.text + print(f'got {url}') + response = requests.get(url) + if response.status_code < 200 or response.status_code >= 300: + startupError(f"Cannot load boostrap bundle (response status: {response.status_code})") + importBundle(BytesIO(response.content)) + + sites_json = f"{data_dir}/sites.json" if not os.path.isfile(sites_json): with open(sites_json, "w") as f: f.write("{}") - users_json = f"{config.data_dir}/users.json" + users_json = f"{data_dir}/users.json" if not os.path.isfile(users_json): with open(users_json, "w") as f: f.write("{}") # TODO: GET RID OF TOP-LEVEL CODE!!! +config.initConsoleLogger() try: init_dirs() @@ -73,7 +121,7 @@ if config.action == "main": r = proc.wait() sys.exit(r) -config.initLogging() +config.initLogging(console_logging=False) # Debug dependent configuration from Debug import DebugHook @@ -414,6 +462,9 @@ class Actions(object): else: return res + def importBundle(self, bundle): + importBundle(bundle) + def getWebsocket(self, site): import websocket From 2b51e23650a82fc24cd52680a3979fe1833a66bb Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 24 Jul 2023 08:21:31 +0000 Subject: [PATCH 14/46] Fix importBundle Avoid using SiteManager as initializing it out of order breaks things --- src/main.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main.py b/src/main.py index b99e0076..660bf362 100644 --- a/src/main.py +++ b/src/main.py @@ -37,8 +37,14 @@ load_config() def importBundle(bundle): from zipfile import ZipFile from Crypt.CryptBitcoin import isValidAddress - from Site import SiteManager - SiteManager.site_manager.load() + import json + + sites_json_path = f"{config.data_dir}/sites.json" + try: + with open(sites_json_path) as f: + sites = json.load(f) + except Exception as err: + sites = {} with ZipFile(bundle) as zf: all_files = zf.namelist() @@ -59,10 +65,11 @@ def importBundle(bundle): info.filename = tgt zf.extract(info) logging.info(f'add site {d}') - SiteManager.site_manager.add(d, noload=True) + sites[d] = {} else: logging.info(f'Warning: unknown file in a bundle: {prefix+d}') - SiteManager.site_manager.save() + with open(sites_json_path, 'w') as f: + json.dump(sites, f) def init_dirs(): data_dir = config.data_dir From 41c7bd47c861eadc0beb58ad551db23fae63a104 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 24 Jul 2023 08:22:19 +0000 Subject: [PATCH 15/46] Use Syncronite directly by default --- src/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.py b/src/Config.py index 8850679b..42d0721c 100644 --- a/src/Config.py +++ b/src/Config.py @@ -433,7 +433,7 @@ class Config(object): self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/caryoscelus/zeronet-conservancy/bootstrap/bootstrap.url', type=str) self.parser.add_argument('--disable_bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true') self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*') - self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', metavar='path', nargs='*') + self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', default=['{data_dir}/15CEFKBRHFfAP9rmL6hhLmHoXrrgmw4B5o/cache/1/Syncronite.html'], metavar='path', nargs='*') self.parser.add_argument('--trackers_proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable") self.parser.add_argument('--use_libsecp256k1', help='Use Libsecp256k1 liblary for speedup', type='bool', choices=[True, False], default=True) self.parser.add_argument('--use_openssl', help='Use OpenSSL liblary for speedup', type='bool', choices=[True, False], default=True) From 1d2e264a44765defcc27890bc1986dbcc570f0c1 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 24 Jul 2023 08:36:00 +0000 Subject: [PATCH 16/46] Remove manually copied tracker list --- src/Config.py | 174 +------------------------------------------------- 1 file changed, 1 insertion(+), 173 deletions(-) diff --git a/src/Config.py b/src/Config.py index 42d0721c..bb779b16 100644 --- a/src/Config.py +++ b/src/Config.py @@ -9,178 +9,6 @@ import logging.handlers import stat import time -trackers = [ - 'zero://188.242.242.224:26474', - 'zero://2001:19f0:8001:1d2f:5400:2ff:fe83:5bf7:23141', - 'zero://200:1e7a:5100:ef7c:6fa4:d8ae:b91c:a74:15441', - 'zero://23.184.48.134:15441', - 'zero://57hzgtu62yzxqgbvgxs7g3lfck3za4zrda7qkskar3tlak5recxcebyd.onion:15445', - 'zero://6i54dd5th73oelv636ivix6sjnwfgk2qsltnyvswagwphub375t3xcad.onion:15441', - 'zero://f2hnjbggc3c2u2apvxdugirnk6bral54ibdoul3hhvu7pd4fso5fq3yd.onion:15441', - 'zero://gugt43coc5tkyrhrc3esf6t6aeycvcqzw7qafxrjpqbwt4ssz5czgzyd.onion:15441', - 'zero://k5w77dozo3hy5zualyhni6vrh73iwfkaofa64abbilwyhhd3wgenbjqd.onion:15441', - 'zero://ow7in4ftwsix5klcbdfqvfqjvimqshbm2o75rhtpdnsderrcbx74wbad.onion:15441', - 'zero://pn4q2zzt2pw4nk7yidxvsxmydko7dfibuzxdswi6gu6ninjpofvqs2id.onion:15441', - 'zero://skdeywpgm5xncpxbbr4cuiip6ey4dkambpanog6nruvmef4f3e7o47qd.onion:15441', - 'zero://wlxav3szbrdhest4j7dib2vgbrd7uj7u7rnuzg22cxbih7yxyg2hsmid.onion:15441', - 'zero://zy7wttvjtsijt5uwmlar4yguvjc2gppzbdj4v6bujng6xwjmkdg7uvqd.onion:15441', - 'http://bt.okmp3.ru:2710/announce', - 'http://fxtt.ru:80/announce', - 'http://incine.ru:6969/announce', - 'http://moeweb.pw:6969/announce', - 'http://open.acgnxtracker.com:80/announce', - 'http://t.acg.rip:6699/announce', - 'http://t.nyaatracker.com:80/announce', - 'http://t.overflow.biz:6969/announce', - 'http://tracker.files.fm:6969/announce', - 'http://tracker.mywaifu.best:6969/announce', - 'http://tracker.vrpnet.org:6969/announce', - 'http://vps02.net.orel.ru:80/announce', - 'udp://960303.xyz:6969/announce', - 'udp://aarsen.me:6969/announce', - 'udp://astrr.ru:6969/announce', - 'udp://ben.kerbertools.xyz:6969/announce', - 'udp://bt1.archive.org:6969/announce', - 'udp://bt2.archive.org:6969/announce', - 'udp://bt.ktrackers.com:6666/announce', - 'udp://bubu.mapfactor.com:6969/announce', - 'udp://c.ns.cluefone.com:6969/announce', - 'udp://cutscloud.duckdns.org:6969/announce', - 'udp://download.nerocloud.me:6969/announce', - 'udp://epider.me:6969/announce', - 'udp://exodus.desync.com:6969/announce', - 'udp://htz3.noho.st:6969/announce', - 'udp://ipv4.tracker.harry.lu:80/announce', - 'udp://laze.cc:6969/announce', - 'udp://mail.artixlinux.org:6969/announce', - 'udp://mirror.aptus.co.tz:6969/announce', - 'udp://moonburrow.club:6969/announce', - 'udp://movies.zsw.ca:6969/announce', - 'udp://mts.tvbit.co:6969/announce', - 'udp://new-line.net:6969/announce', - 'udp://open.demonii.com:1337/announce', - 'udp://open.stealth.si:80/announce', - 'udp://opentracker.i2p.rocks:6969/announce', - 'udp://p4p.arenabg.com:1337/announce', - 'udp://psyco.fr:6969/announce', - 'udp://public.publictracker.xyz:6969/announce', - 'udp://rep-art.ynh.fr:6969/announce', - 'udp://run.publictracker.xyz:6969/announce', - 'udp://sanincode.com:6969/announce', - 'udp://slicie.icon256.com:8000/announce', - 'udp://tamas3.ynh.fr:6969/announce', - 'udp://thouvenin.cloud:6969/announce', - 'udp://torrentclub.space:6969/announce', - 'udp://tracker.0x.tf:6969/announce', - 'udp://tracker1.bt.moack.co.kr:80/announce', - 'udp://tracker.4.babico.name.tr:3131/announce', - 'udp://tracker.altrosky.nl:6969/announce', - 'udp://tracker.artixlinux.org:6969/announce', - 'udp://tracker.farted.net:6969/announce', - 'udp://tracker.jonaslsa.com:6969/announce', - 'udp://tracker.joybomb.tw:6969/announce', - 'udp://tracker.monitorit4.me:6969/announce', - 'udp://tracker.opentrackr.org:1337/announce', - 'udp://tracker.pomf.se:80/announce', - 'udp://tracker.publictracker.xyz:6969/announce', - 'udp://tracker.srv00.com:6969/announce', - 'udp://tracker.tcp.exchange:6969/announce', - 'udp://tracker.theoks.net:6969/announce', - 'udp://transkaroo.joustasie.net:6969/announce', - 'udp://uploads.gamecoast.net:6969/announce', - 'udp://v2.iperson.xyz:6969/announce', - 'udp://vibe.sleepyinternetfun.xyz:1738/announce', - 'udp://www.skynetcenter.me:6969/announce', - 'udp://www.torrent.eu.org:451/announce', - 'zero://194.5.98.39:15441', - 'zero://145.239.95.38:15441', - 'zero://178.128.34.249:26117', - 'zero://217.18.217.143:39288', - 'zero://83.246.141.203:22207', - 'zero://syncronite.loki:15441', - 'zero://2a05:dfc1:4000:1e00::a:15441', - 'zero://2400:6180:100:d0::8fd:8001:21697', - 'zero://2001:19f0:8001:1d2f:5400:2ff:fe83:5bf7:30530', - 'zero://73pyhfwfwsrhfw76knkjfnw6o3lk53zfo7hlxdmxbj75sjcnol5cioad.onion:15442', - 'zero://fzlzmxuz2bust72cuy5g4w6d62tx624xcjaupf2kp7ffuitbiniy2hqd.onion:15441', - 'zero://rlcjomszyitxpwv7kzopmqgzk3bdpsxeull4c3s6goszkk6h2sotfoad.onion:15441', - 'zero://tqmo2nffqo4qc5jgmz3me5eri3zpgf3v2zciufzmhnvznjve5c3argad.onion:15441', - 'http://107.189.31.134:6969/announce', - 'http://119.28.71.45:8080/announce', - 'http://129.146.193.240:6699/announce', - 'http://159.69.65.157:6969/announce', - 'http://163.172.29.130:80/announce', - 'http://185.130.47.2:6969/announce', - 'http://45.67.35.111:6969/announce', - 'http://61.222.178.254:6969/announce', - 'http://83.31.30.182:6969/announce', - 'http://93.158.213.92:1337/announce', - 'http://95.217.167.10:6969/announce', - 'udp://102.223.180.235:6969/announce', - 'udp://103.122.21.50:6969/announce', - 'udp://104.131.98.232:6969/announce', - 'udp://104.244.77.87:6969/announce', - 'udp://107.189.11.58:6969/announce', - 'udp://107.189.31.134:6969/announce', - 'udp://139.144.68.88:6969/announce', - 'udp://149.28.239.70:6969/announce', - 'udp://15.204.205.14:6969/announce', - 'udp://156.234.201.18:80/announce', - 'udp://158.101.161.60:3131/announce', - 'udp://163.172.29.130:80/announce', - 'udp://167.99.185.219:6969/announce', - 'udp://176.31.250.174:6969/announce', - 'udp://176.56.4.238:6969/announce', - 'udp://178.32.222.98:3391/announce', - 'udp://184.105.151.166:6969/announce', - 'udp://185.102.219.163:6969/announce', - 'udp://185.181.60.155:80/announce', - 'udp://185.217.199.21:6969/announce', - 'udp://185.44.82.25:1337/announce', - 'udp://185.68.21.244:6969/announce', - 'udp://192.3.165.191:6969/announce', - 'udp://192.3.165.198:6969/announce', - 'udp://192.95.46.115:6969/announce', - 'udp://193.176.158.162:6969/announce', - 'udp://193.37.214.12:6969/announce', - 'udp://193.42.111.57:9337/announce', - 'udp://198.100.149.66:6969/announce', - 'udp://20.100.205.229:6969/announce', - 'udp://207.241.226.111:6969/announce', - 'udp://207.241.231.226:6969/announce', - 'udp://209.141.59.16:6969/announce', - 'udp://212.237.53.230:6969/announce', - 'udp://23.153.248.2:6969/announce', - 'udp://23.254.228.89:6969/announce', - 'udp://37.187.111.136:6969/announce', - 'udp://37.27.4.53:6969/announce', - 'udp://38.7.201.142:6969/announce', - 'udp://45.154.253.6:6969/announce', - 'udp://45.63.30.114:6969/announce', - 'udp://45.9.60.30:6969/announce', - 'udp://46.38.238.105:6969/announce', - 'udp://49.12.76.8:8080/announce', - 'udp://5.102.159.190:6969/announce', - 'udp://5.196.89.204:6969/announce', - 'udp://51.15.79.209:6969/announce', - 'udp://51.159.54.68:6666/announce', - 'udp://51.68.174.87:6969/announce', - 'udp://51.81.222.188:6969/announce', - 'udp://52.58.128.163:6969/announce', - 'udp://61.222.178.254:6969/announce', - 'udp://77.73.69.230:6969/announce', - 'udp://83.102.180.21:80/announce', - 'udp://83.31.30.182:6969/announce', - 'udp://85.206.172.159:6969/announce', - 'udp://85.239.33.28:6969/announce', - 'udp://86.57.161.157:6969/announce', - 'udp://91.216.110.52:451/announce', - 'udp://93.158.213.92:1337/announce', - 'udp://94.103.87.87:6969/announce', - 'udp://95.216.74.39:6969/announce', - 'udp://95.31.11.224:6969/announce', -] - class Config(object): def __init__(self, argv): @@ -432,7 +260,7 @@ class Config(object): self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip') self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/caryoscelus/zeronet-conservancy/bootstrap/bootstrap.url', type=str) self.parser.add_argument('--disable_bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true') - self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*') + self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=[], metavar='protocol://address', nargs='*') self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', default=['{data_dir}/15CEFKBRHFfAP9rmL6hhLmHoXrrgmw4B5o/cache/1/Syncronite.html'], metavar='path', nargs='*') self.parser.add_argument('--trackers_proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable") self.parser.add_argument('--use_libsecp256k1', help='Use Libsecp256k1 liblary for speedup', type='bool', choices=[True, False], default=True) From 70f2eeada1e30ac893404109c4a0584693fafde5 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 24 Jul 2023 09:18:42 +0000 Subject: [PATCH 17/46] Don't spam console with failed announcements These should be debug, not warning (previously warning reporting was disabled) --- src/Site/SiteAnnouncer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index 2fd63e82..1c4252ae 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -202,7 +202,7 @@ class SiteAnnouncer(object): else: raise AnnounceError("Unknown protocol: %s" % address_parts["protocol"]) except Exception as err: - self.site.log.warning("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode)) + self.site.log.debug("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode)) error = err if error: From d924e9bb2ec0afc71c156a94229cd3fd3c1e98a9 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 24 Jul 2023 10:29:25 +0000 Subject: [PATCH 18/46] fix error handling (was: unbound local variable) --- src/Site/SiteStorage.py | 6 ++++-- src/main.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index 4cbed75d..be8f88e9 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -452,19 +452,21 @@ class SiteStorage(object): bad_files.append(file_inner_path) continue + error = None if quick_check: ok = os.path.getsize(file_path) == content["files"][file_relative_path]["size"] if not ok: - err = "Invalid size" + error = "Invalid size" else: try: ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb")) except Exception as err: + error = err ok = False if not ok: back["num_file_invalid"] += 1 - self.log.debug("[INVALID] %s: %s" % (file_inner_path, err)) + self.log.debug("[INVALID] %s: %s" % (file_inner_path, error)) if add_changed or content.get("cert_user_id"): # If updating own site only add changed user files bad_files.append(file_inner_path) diff --git a/src/main.py b/src/main.py index 57735ae2..181d883b 100644 --- a/src/main.py +++ b/src/main.py @@ -272,18 +272,19 @@ class Actions(object): for content_inner_path in site.content_manager.contents: s = time.time() logging.info("Verifing %s signature..." % content_inner_path) - err = None + error = None try: file_correct = site.content_manager.verifyFile( content_inner_path, site.storage.open(content_inner_path, "rb"), ignore_same=False ) except Exception as err: file_correct = False + error = err if file_correct is True: logging.info("[OK] %s (Done in %.3fs)" % (content_inner_path, time.time() - s)) else: - logging.error("[ERROR] %s: invalid file: %s!" % (content_inner_path, err)) + logging.error("[ERROR] %s: invalid file: %s!" % (content_inner_path, error)) input("Continue?") bad_files += content_inner_path From 433b32417386d46d36025e6dc159929065bb710c Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Tue, 25 Jul 2023 15:11:11 +0000 Subject: [PATCH 19/46] windows os .exe build instruction --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 3d5a64f2..0716c869 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,13 @@ Install autoconf and other basic development tools, python3 and pip, then procee - [for full tor anonymity launch this instead] `python zeronet.py --tor_proxy 127.0.0.1:9150 --tor_controller 127.0.0.1:9151 --tor always` - navigate to http://127.0.0.1:43110 in your favourite browser! +To build .exe + +- follow the same steps as above, but additionally +- `pip install pyinstaller` +- `pyinstaller -p src -p plugins --hidden-import merkletools --hidden-import lib.bencode_open --hidden-import Crypt.Crypt --hidden-import Db.DbQuery --hidden-import lib.subtl --hidden-import lib.subtl.subtl --hidden-import sockshandler --add-data "src;src" --add-data "plugins;plugins" --clean zeronet.py` +- dist/zeronet should contain working zeronet.exe! + ## Current limitations * File transactions are not compressed From a8c2117a55c68605c1d91fc030caefb4e522d45f Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Tue, 25 Jul 2023 15:11:33 +0000 Subject: [PATCH 20/46] CHANGELOG & revision bump --- CHANGELOG.md | 5 +++++ src/Config.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9186fbb5..32a7ac44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ ### zeronet-conservancy 0.7.9+ - update merkletools dependency to avoid legacy pysha3 (@caryoscelus) - fix ReDoS in file editor (UiFileManager plugin) due to outdated codemirror (@caryoscelus) +- more anonymous UPnP (thanks to 0netdwf for reporting) +- remove old zeroname plugins (.bit deprecation) +- sitePublish --recursive option +- windows build instruction improvement +- various improvements ### zeronet-conservancy 0.7.9 (2023-07-02) (f966a4203fe33bd9f35) maintainers: @caryoscelus -> none diff --git a/src/Config.py b/src/Config.py index 8a101cde..fc30a924 100644 --- a/src/Config.py +++ b/src/Config.py @@ -187,7 +187,7 @@ class Config(object): self.version = "0.7.9+" self.user_agent = "conservancy" # DEPRECATED ; replace with git-generated commit - self.rev = 5110 + self.rev = 5115 self.user_agent_rev = 8192 self.argv = argv self.action = None From 0811902ff6849320b19bcb336e748f64a66eb409 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Tue, 25 Jul 2023 18:59:09 +0000 Subject: [PATCH 21/46] Disable third-party access to 0net server. This previously enabled clearnet sites to detect if user is running 0net instance on their machine as well as to detect which 0net sites are downloaded. Check online at https://riza-committee.github.io/demos/0scan.html Intra-0net version of this is still available at http://127.0.0.1:43110/1ScanCY9fjmjanDt7NwvyNQCL16hqWnVM/ --- src/Ui/UiRequest.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 281a5e5c..8f1e4c18 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -282,13 +282,17 @@ class UiRequest(object): # Send response headers def sendHeader(self, status=200, content_type="text/html", noscript=False, allow_ajax=False, script_nonce=None, extra_headers=[]): + ref = self.env.get("HTTP_REFERER") + url = self.getRequestUrl() + if status != 404 and ref and not self.isSameHost(ref, url): + # pretend nothing is here for third-party access + return self.error404() + headers = {} headers["Version"] = "HTTP/1.1" headers["Connection"] = "Keep-Alive" headers["Keep-Alive"] = "max=25, timeout=30" headers["X-Frame-Options"] = "SAMEORIGIN" - if content_type != "text/html" and self.env.get("HTTP_REFERER") and self.isSameOrigin(self.getReferer(), self.getRequestUrl()): - headers["Access-Control-Allow-Origin"] = "*" # Allow load font files from css if noscript: headers["Content-Security-Policy"] = "default-src 'none'; sandbox allow-top-navigation allow-forms; img-src *; font-src * data:; media-src *; style-src * 'unsafe-inline';" @@ -605,7 +609,23 @@ class UiRequest(object): self.server.add_nonces.append(add_nonce) return add_nonce + def isSameHost(self, url_a, url_b): + """Check if urls have the same HOST (to prevent leaking resources to clearnet sites)""" + if not url_a or not url_b: + return False + + url_a = url_a.replace("/raw/", "/") + url_b = url_b.replace("/raw/", "/") + + origin_pattern = "http[s]{0,1}://(.*?/).*" + + origin_a = re.sub(origin_pattern, "\\1", url_a) + origin_b = re.sub(origin_pattern, "\\1", url_b) + + return origin_a == origin_b + def isSameOrigin(self, url_a, url_b): + """Check if 0net origin is the same""" if not url_a or not url_b: return False From f336cd02bd9cdc3893741cd9fa110431e5929ebd Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Tue, 25 Jul 2023 20:55:40 +0000 Subject: [PATCH 22/46] More sophisticated detection of cross-site info leak see previous commit for more info --- src/Ui/UiRequest.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 8f1e4c18..b5d1736e 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -282,9 +282,15 @@ class UiRequest(object): # Send response headers def sendHeader(self, status=200, content_type="text/html", noscript=False, allow_ajax=False, script_nonce=None, extra_headers=[]): - ref = self.env.get("HTTP_REFERER") url = self.getRequestUrl() - if status != 404 and ref and not self.isSameHost(ref, url): + referer = self.env.get('HTTP_REFERER') + origin = self.env.get('HTTP_ORIGIN') + fetch_site = self.env.get('HTTP_SEC_FETCH_SITE') + fetch_mode = self.env.get('HTTP_SEC_FETCH_MODE') + not_same_ref = referer and not self.isSameHost(referer, url) + not_same_origin = origin and not self.isSameHost(origin, url) + cross_site_not_navigate = not referer and fetch_site == 'cross-site' and not fetch_mode == 'navigate' + if status != 404 and (not_same_ref or not_same_origin or cross_site_not_navigate): # pretend nothing is here for third-party access return self.error404() From eeaded23f9d3acc587ae93976e6e20ba66be5e7e Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Wed, 26 Jul 2023 07:14:14 +0000 Subject: [PATCH 23/46] Update bootstrap url --- bootstrap.url | 2 +- src/Config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap.url b/bootstrap.url index f23462eb..8b287997 100644 --- a/bootstrap.url +++ b/bootstrap.url @@ -1 +1 @@ -https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.9/data-default-2023-07-23.zip \ No newline at end of file +https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/data-default-2023-07-26.zip \ No newline at end of file diff --git a/src/Config.py b/src/Config.py index 28a6e19e..f971146b 100644 --- a/src/Config.py +++ b/src/Config.py @@ -260,7 +260,7 @@ class Config(object): self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true') self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port') self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip') - self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/caryoscelus/zeronet-conservancy/bootstrap/bootstrap.url', type=str) + self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/zeronet-conservancy/zeronet-conservancy/master/bootstrap.url', type=str) self.parser.add_argument('--disable_bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true') self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=[], metavar='protocol://address', nargs='*') self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', default=['{data_dir}/15CEFKBRHFfAP9rmL6hhLmHoXrrgmw4B5o/cache/1/Syncronite.html'], metavar='path', nargs='*') From 40ae09dca88d813b760e6cf23598f29d29f17eb9 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Wed, 26 Jul 2023 07:56:31 +0000 Subject: [PATCH 24/46] py3.8 compat --- src/main.py | 5 +++-- src/util/compat.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 src/util/compat.py diff --git a/src/main.py b/src/main.py index 7f5c1b33..5c69c884 100644 --- a/src/main.py +++ b/src/main.py @@ -3,6 +3,7 @@ import sys import stat import time import logging +from util.compat import * startup_errors = [] def startupError(msg): @@ -54,12 +55,12 @@ def importBundle(bundle): else: prefix = '' top_2 = list(set(filter(lambda f: len(f)>0, - map(lambda f: f.removeprefix(prefix).split('/')[0], all_files)))) + map(lambda f: removeprefix(f, prefix).split('/')[0], all_files)))) for d in top_2: if isValidAddress(d): logging.info(f'unpack {d} into {config.data_dir}') for fname in filter(lambda f: f.startswith(prefix+d) and not f.endswith('/'), all_files): - tgt = config.data_dir + '/' + fname.removeprefix(prefix) + tgt = config.data_dir + '/' + removeprefix(fname, prefix) logging.info(f'-- {fname} --> {tgt}') info = zf.getinfo(fname) info.filename = tgt diff --git a/src/util/compat.py b/src/util/compat.py new file mode 100644 index 00000000..f41e67b2 --- /dev/null +++ b/src/util/compat.py @@ -0,0 +1,16 @@ +import sys + +if sys.version_info.major == 3 and sys.version_info.minor < 9: + def removeprefix(s, prefix, /): + if s.startswith(prefix): + return s[len(prefix):] + return s + def removesuffix(s, suffix, /): + if s.endswith(suffix): + return s[:-len(suffix)] + return s +else: + def removeprefix(s, prefix, /): + return s.removeprefix(prefix) + def removesuffix(s, suffix, /): + return s.removesuffix(suffix) From 18d35d3bed4f0683e99f8af5a86a8d76ed866e1e Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Wed, 26 Jul 2023 08:01:47 +0000 Subject: [PATCH 25/46] v0.7.10 --- CHANGELOG.md | 8 +++++++- src/Config.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32a7ac44..b228efca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,16 @@ -### zeronet-conservancy 0.7.9+ +### zeronet-conservancy 0.7.10+ + +### zeronet-conservancy 0.7.10 (2023-07-26) +prepared by @caryoscelus - update merkletools dependency to avoid legacy pysha3 (@caryoscelus) - fix ReDoS in file editor (UiFileManager plugin) due to outdated codemirror (@caryoscelus) - more anonymous UPnP (thanks to 0netdwf for reporting) - remove old zeroname plugins (.bit deprecation) - sitePublish --recursive option - windows build instruction improvement +- importBundle command line action +- use Syncronite tracker site by default +- fix leak of local 0net sites to clearnet sites (originally reported by @egosown/beardog) - various improvements ### zeronet-conservancy 0.7.9 (2023-07-02) (f966a4203fe33bd9f35) diff --git a/src/Config.py b/src/Config.py index f971146b..2366b89c 100644 --- a/src/Config.py +++ b/src/Config.py @@ -12,10 +12,10 @@ import time class Config(object): def __init__(self, argv): - self.version = "0.7.9+" + self.version = "0.7.10" self.user_agent = "conservancy" # DEPRECATED ; replace with git-generated commit - self.rev = 5115 + self.rev = 5120 self.user_agent_rev = 8192 self.argv = argv self.action = None From 714729edab7df31030ed6c3db3661f970c8eaaed Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sat, 29 Jul 2023 16:47:11 +0000 Subject: [PATCH 26/46] New development cycle --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b228efca..b59e09fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ### zeronet-conservancy 0.7.10+ -### zeronet-conservancy 0.7.10 (2023-07-26) +### zeronet-conservancy 0.7.10 (2023-07-26) (18d35d3bed4f0683e99) prepared by @caryoscelus - update merkletools dependency to avoid legacy pysha3 (@caryoscelus) - fix ReDoS in file editor (UiFileManager plugin) due to outdated codemirror (@caryoscelus) From e79924d502d5b7d0e1b7354d5d9c4565c8631386 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sat, 29 Jul 2023 17:17:06 +0000 Subject: [PATCH 27/46] Fix downloading GeoIP database fixes #222 --- plugins/Sidebar/SidebarPlugin.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/plugins/Sidebar/SidebarPlugin.py b/plugins/Sidebar/SidebarPlugin.py index c117de0e..b8c5f0f3 100644 --- a/plugins/Sidebar/SidebarPlugin.py +++ b/plugins/Sidebar/SidebarPlugin.py @@ -602,7 +602,7 @@ class UiWebsocketPlugin(object): def downloadGeoLiteDb(self, db_path): import gzip import shutil - from util import helper + import requests if config.offline or config.tor == 'always': return False @@ -617,19 +617,18 @@ class UiWebsocketPlugin(object): downloadl_err = None try: # Download - response = helper.httpRequest(db_url) - data_size = response.getheader('content-length') + response = requests.get(db_url, stream=True) + data_size = response.headers.get('content-length') + if data_size is None: + data.write(response.content) + data_size = int(data_size) data_recv = 0 data = io.BytesIO() - while True: - buff = response.read(1024 * 512) - if not buff: - break + for buff in response.iter_content(chunk_size=1024 * 512): data.write(buff) data_recv += 1024 * 512 - if data_size: - progress = int(float(data_recv) / int(data_size) * 100) - self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], progress]) + progress = int(float(data_recv) / data_size * 100) + self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], progress]) self.log.info("GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell()) data.seek(0) From b358435016231a3f53c37224bbf68860aaf586f9 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sat, 29 Jul 2023 17:20:45 +0000 Subject: [PATCH 28/46] Remove dead code --- src/util/helper.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/src/util/helper.py b/src/util/helper.py index a0daa557..af65f727 100644 --- a/src/util/helper.py +++ b/src/util/helper.py @@ -195,44 +195,6 @@ def mergeDicts(dicts): return dict(back) -# Request https url using gevent SSL error workaround -def httpRequest(url, as_file=False): - if url.startswith("http://"): - import urllib.request - response = urllib.request.urlopen(url) - else: # Hack to avoid Python gevent ssl errors - import socket - import http.client - import ssl - - host, request = re.match("https://(.*?)(/.*?)$", url).groups() - - conn = http.client.HTTPSConnection(host) - sock = socket.create_connection((conn.host, conn.port), conn.timeout, conn.source_address) - - context = ssl.create_default_context() - context.minimum_version = ssl.TLSVersion.TLSv1_2 - - conn.sock = context.wrap_socket(sock, conn.key_file, conn.cert_file) - conn.request("GET", request) - response = conn.getresponse() - if response.status in [301, 302, 303, 307, 308]: - logging.info("Redirect to: %s" % response.getheader('Location')) - response = httpRequest(response.getheader('Location')) - - if as_file: - import io - data = io.BytesIO() - while True: - buff = response.read(1024 * 16) - if not buff: - break - data.write(buff) - return data - else: - return response - - def timerCaller(secs, func, *args, **kwargs): gevent.spawn_later(secs, timerCaller, secs, func, *args, **kwargs) func(*args, **kwargs) From a509032c8e4bbfc5c81c80d44d79660d7eaf060a Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sat, 29 Jul 2023 18:03:30 +0000 Subject: [PATCH 29/46] Drop attempts at py<3.6 compatibility Wake up, it's 2023 now. Also f-strings has been used prominently already --- src/Content/ContentManager.py | 5 +---- src/Db/Db.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index bcf96c0b..a06ba523 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -928,10 +928,7 @@ class ContentManager(object): new_content = file else: try: - if sys.version_info.major == 3 and sys.version_info.minor < 6: - new_content = json.loads(file.read().decode("utf8")) - else: - new_content = json.load(file) + new_content = json.load(file) except Exception as err: raise VerifyError(f"Invalid json file: {err}") if inner_path in self.contents: diff --git a/src/Db/Db.py b/src/Db/Db.py index 3d4b6d7d..6ee3a977 100644 --- a/src/Db/Db.py +++ b/src/Db/Db.py @@ -377,10 +377,7 @@ class Db(object): if file_path.endswith("json.gz"): file = helper.limitedGzipFile(fileobj=file) - if sys.version_info.major == 3 and sys.version_info.minor < 6: - data = json.loads(file.read().decode("utf8")) - else: - data = json.load(file) + data = json.load(file) except Exception as err: self.log.debug("Json file %s load error: %s" % (file_path, err)) data = {} From ebd81dad9d99745e903ed905644101684a16e682 Mon Sep 17 00:00:00 2001 From: slrslr <6596726+slrslr@users.noreply.github.com> Date: Sun, 30 Jul 2023 05:17:15 +0000 Subject: [PATCH 30/46] Update README.md Adding new simplified installation commands. It should allow easier installation and update on most Linux distributions using 2 commands. --- README.md | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 0716c869..0a1f641f 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,14 @@ [по-русски](README-ru.md) | [em português](README-ptbr.md) | [简体中文](README-zh-cn.md) -zeronet-conservancy is a fork/continuation of [ZeroNet](https://github.com/HelloZeroNet/ZeroNet) project +Zeronet-Conservancy is a fork/continuation of [ZeroNet](https://github.com/HelloZeroNet/ZeroNet) project (that has been abandoned by its creator) that is dedicated to sustaining existing p2p network and developing its values of decentralization and freedom, while gradually switching to a better designed network ## No active maintainer warning This fork was created and maintained by @caryoscelus, but due to vanishing interest and in order to avoid having -another one-person project, they stepped down. This means there currently is no active maintainer (you're are -welcome to become one!), however some development might still happen. +another one-person project, the development is limitted. ## Why fork? @@ -75,25 +74,34 @@ Following links relate to original ZeroNet: ### Install from your distribution repository -- NixOS: https://search.nixos.org/packages?channel=22.05&show=zeronet-conservancy&type=packages&query=zeronet-conservancy (and see below) +- NixOS: [zeronet-conservancy packages search](https://search.nixos.org/packages?from=0&size=50&sort=relevance&type=packages&query=zeronet-conservancy) (and see below) - ArchLinux: [latest release](https://aur.archlinux.org/packages/zeronet-conservancy), [fresh git version](https://aur.archlinux.org/packages/zeronet-conservancy-git) ### Install from Nix package manager (Linux or MacOS) -``` -# install & configure nix package manager -nix-env -iA nixpkgs.zeronet-conservancy -``` - +if you're on NixOS, install & configure nix package manager: +```nix-env -iA nixpkgs.zeronet-conservancy``` or - -`nix-env -iA nixos.zeronet-conservancy` - -if you're on NixOS - +```nix-env -iA nixos.zeronet-conservancy``` (thanks @fgaz for making & maintaining the package) -### Install from source +### Install from source - simplified, using Git and Python PIP + +1. Install Git and Python PIP package: + +- Debian and Ubuntu based: `sudo apt install git python3-pip -y` +- Red Hat and Fedora based: `yum install epel-release -y 2>/dev/null;yum install git python3 python3-wheel` +- Fedora based dandified: `sudo dnf install git python3-pip python3-wheel -y` +- Arch and Manjaro based: `sudo pacman -S git python-pip -v --no-confirm` +- openSUSE: `sudo zypper install python3-pip python3-setuptools python3-wheel` + +2. Clone Github repository and install required Python modules. First edit zndir path at the begining of the command, to be the path where you want to store Zeronet-Conservancy: + +`zndir="/home/user/myapps/zeronet" ; if [[ ! -d "$zndir" ]]; then git clone --recursive "https://github.com/zeronet-conservancy/zeronet-conservancy.git" "$zndir" && cd "$zndir"||exit; else cd "$zndir";git pull origin master; fi; cd "$zndir" && pip install -r requirements.txt|grep -v "already satisfied"; echo "Try to run: python3 $(pwd)/zeronet.py"` + +(This command can also be used to keep Zeronet-Conservancy up to date) + +### Install from source - detailed #### System dependencies @@ -143,9 +151,9 @@ Install autoconf and other basic development tools, python3 and pip, then procee - after installing general dependencies and cloning repo (as above), run `start-venv.sh` which will create a virtual env for you and install python requirements - more convenience scripts to be added soon -### Building under windows os +### Building under Windows OS -(this instruction is work-in-progress, please help us test it and improve it!) +(These instructions are work-in-progress, please help us test it and improve it!) - install python from https://www.python.org/downloads/ - install some windows compiler suitable for python , this proved to be the most difficult part for me as non-windows user (see here https://wiki.python.org/moin/WindowsCompilers and i'll link more references later) From a334eefdf733449455c3eb312a02a2b8d0dd1128 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sun, 30 Jul 2023 19:00:43 +0000 Subject: [PATCH 31/46] Bump version --- src/Config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Config.py b/src/Config.py index 2366b89c..e92c653c 100644 --- a/src/Config.py +++ b/src/Config.py @@ -12,10 +12,10 @@ import time class Config(object): def __init__(self, argv): - self.version = "0.7.10" + self.version = "0.7.10+" self.user_agent = "conservancy" # DEPRECATED ; replace with git-generated commit - self.rev = 5120 + self.rev = 5121 self.user_agent_rev = 8192 self.argv = argv self.action = None From 30db5a4652dd65f7de47871dd41161d45de11c34 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 19:01:55 +0700 Subject: [PATCH 32/46] Fix https://github.com/HelloZeroNet/ZeroNet/issues/2757 --- src/util/SafeRe.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/util/SafeRe.py b/src/util/SafeRe.py index 6018e2d3..20827f38 100644 --- a/src/util/SafeRe.py +++ b/src/util/SafeRe.py @@ -15,9 +15,10 @@ def isSafePattern(pattern): if unsafe_pattern_match: raise UnsafePatternError("Potentially unsafe part of the pattern: %s in %s" % (unsafe_pattern_match.group(0), pattern)) - repetitions = re.findall(r"\.[\*\{\+]", pattern) - if len(repetitions) >= 10: - raise UnsafePatternError("More than 10 repetitions of %s in %s" % (repetitions[0], pattern)) + repetitions1 = re.findall(r"\.[\*\{\+]", pattern) + repetitions2 = re.findall(r"[^(][?]", pattern) + if len(repetitions1) + len(repetitions2) >= 10: + raise UnsafePatternError("More than 10 repetitions in %s" % pattern) return True From 5fadd5f9bda78e307e82e52e4f889c2ff755e950 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 31 Jul 2023 08:28:29 +0000 Subject: [PATCH 33/46] Improve SafeRe code readability function isSafePattern was never used as boolean function, its only useful behaviour being raising exception on bad regexp, so it's renamed and reused accordingly --- plugins/Sidebar/ConsolePlugin.py | 5 ++--- src/util/SafeRe.py | 12 ++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/plugins/Sidebar/ConsolePlugin.py b/plugins/Sidebar/ConsolePlugin.py index 15f6a1ba..12d49fbf 100644 --- a/plugins/Sidebar/ConsolePlugin.py +++ b/plugins/Sidebar/ConsolePlugin.py @@ -14,8 +14,7 @@ class WsLogStreamer(logging.StreamHandler): self.ui_websocket = ui_websocket if filter: - if not SafeRe.isSafePattern(filter): - raise Exception("Not a safe prex pattern") + SafeRe.guard(filter): self.filter_re = re.compile(".*" + filter) else: self.filter_re = None @@ -55,7 +54,7 @@ class UiWebsocketPlugin(object): pos_start = log_file.tell() lines = [] if filter: - assert SafeRe.isSafePattern(filter) + SafeRe.guard(filter) filter_re = re.compile(".*" + filter) last_match = False diff --git a/src/util/SafeRe.py b/src/util/SafeRe.py index 20827f38..30aa1f29 100644 --- a/src/util/SafeRe.py +++ b/src/util/SafeRe.py @@ -7,7 +7,8 @@ class UnsafePatternError(Exception): cached_patterns = {} -def isSafePattern(pattern): +def guard(pattern): + '''Checks if pattern is safe and raises exception if it isn't''' if len(pattern) > 255: raise UnsafePatternError("Pattern too long: %s characters in %s" % (len(pattern), pattern)) @@ -20,14 +21,13 @@ def isSafePattern(pattern): if len(repetitions1) + len(repetitions2) >= 10: raise UnsafePatternError("More than 10 repetitions in %s" % pattern) - return True - def match(pattern, *args, **kwargs): + '''Guard for safety, compile, cache and match regexp''' cached_pattern = cached_patterns.get(pattern) if cached_pattern: return cached_pattern.match(*args, **kwargs) else: - if isSafePattern(pattern): - cached_patterns[pattern] = re.compile(pattern) - return cached_patterns[pattern].match(*args, **kwargs) + guard(pattern) + cached_patterns[pattern] = re.compile(pattern) + return cached_patterns[pattern].match(*args, **kwargs) From 7d4e66e07b6ced6bfc9c13cf3e9296c21ac517ef Mon Sep 17 00:00:00 2001 From: slrslr <6596726+slrslr@users.noreply.github.com> Date: Mon, 31 Jul 2023 18:34:11 +0000 Subject: [PATCH 34/46] Update README.md Mention first Windows build available for download. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 0a1f641f..1319b13a 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,10 @@ Install autoconf and other basic development tools, python3 and pip, then procee - after installing general dependencies and cloning repo (as above), run `start-venv.sh` which will create a virtual env for you and install python requirements - more convenience scripts to be added soon +### Windows build + +Download and extract .zip archive [zeronet-conservancy-0.7.10-unofficial-win64.zip](https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/zeronet-conservancy-0.7.10-unofficial-win64.zip) + ### Building under Windows OS (These instructions are work-in-progress, please help us test it and improve it!) From b95b979c7fc31a337fc3ca545bab51a0576a1227 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 10 Aug 2023 20:30:24 +0000 Subject: [PATCH 35/46] README: partially rewrite installing --- README.md | 64 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1319b13a..00ce2fda 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [по-русски](README-ru.md) | [em português](README-ptbr.md) | [简体中文](README-zh-cn.md) -Zeronet-Conservancy is a fork/continuation of [ZeroNet](https://github.com/HelloZeroNet/ZeroNet) project +`zeronet-conservancy` is a fork/continuation of [ZeroNet](https://github.com/HelloZeroNet/ZeroNet) project (that has been abandoned by its creator) that is dedicated to sustaining existing p2p network and developing its values of decentralization and freedom, while gradually switching to a better designed network @@ -79,40 +79,39 @@ Following links relate to original ZeroNet: ### Install from Nix package manager (Linux or MacOS) -if you're on NixOS, install & configure nix package manager: -```nix-env -iA nixpkgs.zeronet-conservancy``` -or -```nix-env -iA nixos.zeronet-conservancy``` + - install & configure nix package manager (if needed) + - `nix-env -iA nixpkgs.zeronet-conservancy` + +or add `zeronet-conservancy` to your system configuration if you're on NixOS + (thanks @fgaz for making & maintaining the package) -### Install from source - simplified, using Git and Python PIP - -1. Install Git and Python PIP package: - -- Debian and Ubuntu based: `sudo apt install git python3-pip -y` -- Red Hat and Fedora based: `yum install epel-release -y 2>/dev/null;yum install git python3 python3-wheel` -- Fedora based dandified: `sudo dnf install git python3-pip python3-wheel -y` -- Arch and Manjaro based: `sudo pacman -S git python-pip -v --no-confirm` -- openSUSE: `sudo zypper install python3-pip python3-setuptools python3-wheel` - -2. Clone Github repository and install required Python modules. First edit zndir path at the begining of the command, to be the path where you want to store Zeronet-Conservancy: - -`zndir="/home/user/myapps/zeronet" ; if [[ ! -d "$zndir" ]]; then git clone --recursive "https://github.com/zeronet-conservancy/zeronet-conservancy.git" "$zndir" && cd "$zndir"||exit; else cd "$zndir";git pull origin master; fi; cd "$zndir" && pip install -r requirements.txt|grep -v "already satisfied"; echo "Try to run: python3 $(pwd)/zeronet.py"` - -(This command can also be used to keep Zeronet-Conservancy up to date) - -### Install from source - detailed +### Install from source #### System dependencies ##### Generic unix-like (including mac os x) Install autoconf and other basic development tools, python3 and pip, then proceed to "building python dependencies" +(if running fails due to missing dependency, please report it/make pull request to fix dependency list) ##### Apt-based (debian, ubuntu, etc) - `sudo apt update` - `sudo apt install pkg-config libffi-dev python3-pip python3-venv python3-dev build-essential` +##### Red Hat and Fedora based + - `yum install epel-release -y 2>/dev/null` + - `yum install git python3 python3-wheel` + +##### Fedora based dandified + - `sudo dnf install git python3-pip python3-wheel -y` + +##### openSUSE + - `sudo zypper install python3-pip python3-setuptools python3-wheel` + +##### Arch and Manjaro based + - `sudo pacman -S git python-pip -v --no-confirm` + ##### Android/Termux - install [Termux](https://termux.com/) (in Termux you can install packages via `pkg install `) - `pkg update` @@ -122,7 +121,7 @@ Install autoconf and other basic development tools, python3 and pip, then procee - (optional) `pkg install tor` - (optional) run tor via `tor --ControlPort 9051 --CookieAuthentication 1` command (you can then open new session by swiping to the right) -#### Building python dependencies venv & running +#### Building python dependencies, venv & running - clone this repo (NOTE: on Android/Termux you should clone it into "home" folder of Termux, because virtual environment cannot live in `storage/`) - `python3 -m venv venv` (make python virtual environment, the last `venv` is just a name, if you use different you should replace it in later commands) - `source venv/bin/activate` (activate environment) @@ -147,13 +146,26 @@ Install autoconf and other basic development tools, python3 and pip, then procee - or: `docker compose up -d 0net-tor` for run 0net and tor in one container. (please check if these instructions are still accurate) +#### Alternative one-liner (installing python dependencies globally) + +Clone Github repository and install required Python modules. First +edit zndir path at the begining of the command, to be the path where +you want to store `zeronet-conservancy`: + +`zndir="/home/user/myapps/zeronet" ; if [[ ! -d "$zndir" ]]; then git clone --recursive "https://github.com/zeronet-conservancy/zeronet-conservancy.git" "$zndir" && cd "$zndir"||exit; else cd "$zndir";git pull origin master; fi; cd "$zndir" && pip install -r requirements.txt|grep -v "already satisfied"; echo "Try to run: python3 $(pwd)/zeronet.py"` + +(This command can also be used to keep `zeronet-conservancy` up to date) + #### Alternative script - - after installing general dependencies and cloning repo (as above), run `start-venv.sh` which will create a virtual env for you and install python requirements + - after installing general dependencies and cloning repo (as above), + run `start-venv.sh` which will create a virtual env for you and + install python requirements - more convenience scripts to be added soon -### Windows build +### (unofficial) Windows OS build -Download and extract .zip archive [zeronet-conservancy-0.7.10-unofficial-win64.zip](https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/zeronet-conservancy-0.7.10-unofficial-win64.zip) +Download and extract .zip archive +[zeronet-conservancy-0.7.10-unofficial-win64.zip](https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/zeronet-conservancy-0.7.10-unofficial-win64.zip) ### Building under Windows OS From 4153f60258ef9d941e21eede2973ec69f3fd17be Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Fri, 18 Aug 2023 16:12:31 +0000 Subject: [PATCH 36/46] Don't accept file update if it's from muted user refs https://github.com/HelloZeroNet/ZeroNet/issues/1855 --- plugins/ContentFilter/ContentFilterPlugin.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/ContentFilter/ContentFilterPlugin.py b/plugins/ContentFilter/ContentFilterPlugin.py index 6cec1bc3..8a9db880 100644 --- a/plugins/ContentFilter/ContentFilterPlugin.py +++ b/plugins/ContentFilter/ContentFilterPlugin.py @@ -216,6 +216,18 @@ class SiteStoragePlugin(object): filter_storage.includeUpdateAll() return super(SiteStoragePlugin, self).onUpdated(inner_path, file=file) +@PluginManager.registerTo("FileRequest") +class FileRequestPlugin: + def actionUpdate(self, params): + inner_path = params.get('inner_path', '') + self.log.info(f'FileRequest.actionUpdate {inner_path}') + matches = re.findall('/(1[A-Za-z0-9]{26,35})/', inner_path) + for auth_address in matches: + if filter_storage.isMuted(auth_address): + self.log.info(f'Mute match in FileRequest.actionUpdate: {auth_address}, ignoring {inner_path}') + self.response({'ok': f'Thanks, file {inner_path} updated!'}) + return False + return super(FileRequestPlugin, self).actionUpdate(params) @PluginManager.registerTo("UiRequest") class UiRequestPlugin(object): From 4d4880a5c8d94d85900898f9742f32d2e4e70812 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Fri, 18 Aug 2023 16:51:16 +0000 Subject: [PATCH 37/46] Ignore muted files in Site.needFile --- plugins/ContentFilter/ContentFilterPlugin.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/plugins/ContentFilter/ContentFilterPlugin.py b/plugins/ContentFilter/ContentFilterPlugin.py index 8a9db880..ec88b3b3 100644 --- a/plugins/ContentFilter/ContentFilterPlugin.py +++ b/plugins/ContentFilter/ContentFilterPlugin.py @@ -216,11 +216,21 @@ class SiteStoragePlugin(object): filter_storage.includeUpdateAll() return super(SiteStoragePlugin, self).onUpdated(inner_path, file=file) +@PluginManager.registerTo("Site") +class SitePlugin(object): + def needFile(self, inner_path, update=False, blocking=True, peer=None, priority=0): + self.log.debug(f'needFile {inner_path}') + matches = re.findall('/(1[A-Za-z0-9]{26,35})/', inner_path) + for auth_address in matches: + if filter_storage.isMuted(auth_address): + self.log.info(f'Mute match in Site.needFile: {auth_address}, ignoring {inner_path}') + return False + return super(SitePlugin, self).needFile(inner_path, update, blocking, peer, priority) + @PluginManager.registerTo("FileRequest") class FileRequestPlugin: def actionUpdate(self, params): inner_path = params.get('inner_path', '') - self.log.info(f'FileRequest.actionUpdate {inner_path}') matches = re.findall('/(1[A-Za-z0-9]{26,35})/', inner_path) for auth_address in matches: if filter_storage.isMuted(auth_address): From 21eb421a8f995c5ad6f75f5354e376832efc714d Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 4 Sep 2023 10:26:37 +0000 Subject: [PATCH 38/46] Update data-default --- bootstrap.url | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.url b/bootstrap.url index 8b287997..ec7a3944 100644 --- a/bootstrap.url +++ b/bootstrap.url @@ -1 +1 @@ -https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/data-default-2023-07-26.zip \ No newline at end of file +https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/data-default-2023-09-03.zip \ No newline at end of file From 598b8222554812d64e12ab93d4a013f1e5ff9b5c Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 4 Sep 2023 21:25:03 +0000 Subject: [PATCH 39/46] Fix typo in Sidebar/ConsolePlugin --- plugins/Sidebar/ConsolePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/Sidebar/ConsolePlugin.py b/plugins/Sidebar/ConsolePlugin.py index 12d49fbf..46cd9aca 100644 --- a/plugins/Sidebar/ConsolePlugin.py +++ b/plugins/Sidebar/ConsolePlugin.py @@ -14,7 +14,7 @@ class WsLogStreamer(logging.StreamHandler): self.ui_websocket = ui_websocket if filter: - SafeRe.guard(filter): + SafeRe.guard(filter) self.filter_re = re.compile(".*" + filter) else: self.filter_re = None From 337f98a479aae4f8ab1424edf209a94274c125ec Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 4 Sep 2023 21:39:46 +0000 Subject: [PATCH 40/46] Add --disable_port_check option --- src/Config.py | 1 + src/File/FileServer.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Config.py b/src/Config.py index e92c653c..1f62b184 100644 --- a/src/Config.py +++ b/src/Config.py @@ -256,6 +256,7 @@ class Config(object): self.parser.add_argument('--ip_local', help='My local ips', default=ip_local, type=int, metavar='ip', nargs='*') self.parser.add_argument('--ip_external', help='Set reported external ip (tested on start if None)', metavar='ip', nargs='*') self.parser.add_argument('--offline', help='Disable network communication', action='store_true') + self.parser.add_argument('--disable_port_check', help='Disable checking port', action='store_true') self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true') self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port') diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 41f76817..d1de4761 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -152,9 +152,13 @@ class FileServer(ConnectionServer): FileRequest = imp.load_source("FileRequest", "src/File/FileRequest.py").FileRequest def portCheck(self): - if config.offline or config.tor == 'always': - msg = "Offline mode" if config.offline else "Tor-only" - self.log.info(f'{msg}: port check disabled') + if config.offline or config.tor == 'always' or config.disable_port_check: + if config.offline: + self.log.info(f'Offline mode: port check disabled') + elif config.tor == 'always': + self.log.info('Tor-only mode: port check disabled') + else: + self.log.info('Port check disabled') res = {"ipv4": None, "ipv6": None} self.port_opened = res return res From 77a70e513dffaabbca10cd0659f28dac3d599e2f Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 7 Sep 2023 20:09:33 +0000 Subject: [PATCH 41/46] README: attribution --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 00ce2fda..59fa72f0 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ Install autoconf and other basic development tools, python3 and pip, then procee - or: `docker compose up -d 0net-tor` for run 0net and tor in one container. (please check if these instructions are still accurate) -#### Alternative one-liner (installing python dependencies globally) +#### Alternative one-liner (by @ssdifnskdjfnsdjk) (installing python dependencies globally) Clone Github repository and install required Python modules. First edit zndir path at the begining of the command, to be the path where From 1eb094bba50fd08d5fba74cba86902f953630815 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 14 Sep 2023 11:03:23 +0000 Subject: [PATCH 42/46] Fix detection of non-writeable start directory --- src/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.py b/src/Config.py index e92c653c..d5e459e7 100644 --- a/src/Config.py +++ b/src/Config.py @@ -71,7 +71,7 @@ class Config(object): elif this_file.endswith("/core/src/Config.py"): # Running as exe or source is at Application Support directory, put var files to outside of core dir start_dir = this_file.replace("/core/src/Config.py", "") - elif this_file.endswith("usr/share/zeronet/src/Config.py"): + elif not os.access(this_file.replace('/src/Config.py', ''), os.R_OK | os.W_OK): # Running from non-writeable location, e.g., AppImage start_dir = os.path.expanduser("~/ZeroNet") else: From 537d7337e2aa0dcd9a5917e6c341a18d2b9c2895 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Fri, 15 Sep 2023 14:23:19 +0000 Subject: [PATCH 43/46] debug log --- src/Worker/Worker.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Worker/Worker.py b/src/Worker/Worker.py index b7111ba1..9adba30d 100644 --- a/src/Worker/Worker.py +++ b/src/Worker/Worker.py @@ -7,6 +7,8 @@ from Debug import Debug from Config import config from Content.ContentManager import VerifyError +import traceback + class WorkerDownloadError(Exception): pass @@ -119,13 +121,15 @@ class Worker(object): self.manager.log.error("%s: Error writing: %s (%s: %s)" % (self.key, task["inner_path"], type(err), err)) raise WorkerIOError(str(err)) - def onTaskVerifyFail(self, task, error_message): + def onTaskVerifyFail(self, task, error): self.num_failed += 1 if self.manager.started_task_num < 50 or config.verbose: - self.manager.log.debug( + self.manager.log.info( "%s: Verify failed: %s, error: %s, failed peers: %s, workers: %s" % - (self.key, task["inner_path"], error_message, len(task["failed"]), task["workers_num"]) + (self.key, task["inner_path"], error, len(task["failed"]), task["workers_num"]) ) + # traceback.format_ + self.manager.log.debug(''.join(traceback.format_exception(error))) task["failed"].append(self.peer) self.peer.hash_failed += 1 if self.peer.hash_failed >= max(len(self.manager.tasks), 3) or self.peer.connection_error > 10: From c9dbc49375e6c17579251982fc6421db3a120d23 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sat, 16 Sep 2023 12:27:00 +0000 Subject: [PATCH 44/46] Delete muted content --- plugins/ContentFilter/ContentFilterPlugin.py | 2 +- plugins/ContentFilter/ContentFilterStorage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/ContentFilter/ContentFilterPlugin.py b/plugins/ContentFilter/ContentFilterPlugin.py index ec88b3b3..c4d8bf3d 100644 --- a/plugins/ContentFilter/ContentFilterPlugin.py +++ b/plugins/ContentFilter/ContentFilterPlugin.py @@ -58,7 +58,7 @@ class UiWebsocketPlugin(object): def actionMuteAdd(self, to, auth_address, cert_user_id, reason): self.cmd( "prompt", - [_["Hide all content from %s?"] % html.escape(cert_user_id), reason, _["Mute"]], + [_["Remove all content from %s?"] % html.escape(cert_user_id), reason, _["Mute"]], lambda res: self.cbMuteAdd(to, auth_address, cert_user_id, res if res else reason) ) diff --git a/plugins/ContentFilter/ContentFilterStorage.py b/plugins/ContentFilter/ContentFilterStorage.py index 289ec2a9..2ad378d6 100644 --- a/plugins/ContentFilter/ContentFilterStorage.py +++ b/plugins/ContentFilter/ContentFilterStorage.py @@ -158,7 +158,7 @@ class ContentFilterStorage(object): dir_inner_path = helper.getDirname(row["inner_path"]) for file_name in site.storage.walk(dir_inner_path): if action == "remove": - site.storage.onUpdated(dir_inner_path + file_name, False) + site.storage.delete(dir_inner_path + file_name) else: site.storage.onUpdated(dir_inner_path + file_name) site.onFileDone(dir_inner_path + file_name) From 19056b408ad9c956c70e4c0237c1ab8e17955228 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 21 Sep 2023 15:28:05 +0000 Subject: [PATCH 45/46] Update gevent to 23.9.0 https://security.snyk.io/vuln/SNYK-PYTHON-GEVENT-5906371 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 887138d8..1059795c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability -gevent>=20.9.0 +gevent>=23.9.0 msgpack>=0.6.0 base58 # for some reason nobody released fresh merkletools that don't require on outdated pysha3 From c92b8bc56c796a962baec69f42072433c65bb3b7 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Wed, 1 Nov 2023 22:12:14 +0000 Subject: [PATCH 46/46] Fix UiServer.getPosted hanging in some circumstances fixes #198 while it's not exactly clear what causes the difference in behaviour, but under certain conditions UiServer.getPosted used to hang trying to readline() POST request (e.g. from UiPassword login). using read(CONTENT_LENGTH) seems to fix the issue --- src/Ui/UiRequest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index b5d1736e..0f8437d6 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -231,8 +231,12 @@ class UiRequest(object): # Return: Posted variables def getPosted(self): if self.env['REQUEST_METHOD'] == "POST": + try: + content_length = int(self.env.get('CONTENT_LENGTH', 0)) + except ValueError: + content_length = 0 return dict(urllib.parse.parse_qsl( - self.env['wsgi.input'].readline().decode() + self.env['wsgi.input'].read(content_length).decode() )) else: return {}