From 260c4e626bc9021ae3e92d80b920ff9fa44c0fc2 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 20 Jul 2023 18:28:04 +0000 Subject: [PATCH 01/12] make tests launch --- src/Test/conftest.py | 3 --- src/Test/pytest.ini | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Test/conftest.py b/src/Test/conftest.py index c8739086..57c61f3a 100644 --- a/src/Test/conftest.py +++ b/src/Test/conftest.py @@ -63,9 +63,6 @@ config.debug = True os.chdir(os.path.abspath(os.path.dirname(__file__) + "/../..")) # Set working dir -all_loaded = PluginManager.plugin_manager.loadPlugins() -assert all_loaded, "Not all plugin loaded successfully" - config.loadPlugins() config.parse(parse_config=False) # Parse again to add plugin configuration options diff --git a/src/Test/pytest.ini b/src/Test/pytest.ini index 556389a2..0ffb385f 100644 --- a/src/Test/pytest.ini +++ b/src/Test/pytest.ini @@ -1,6 +1,6 @@ [pytest] python_files = Test*.py -addopts = -rsxX -v --durations=6 --no-print-logs --capture=fd +addopts = -rsxX -v --durations=6 --capture=fd markers = slow: mark a tests as slow. webtest: mark a test as a webtest. From 51a6eaa057f07deda864355fd2bc62d19072f1a0 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 20 Jul 2023 18:28:21 +0000 Subject: [PATCH 02/12] fix threadpool test --- src/Test/TestThreadPool.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/Test/TestThreadPool.py b/src/Test/TestThreadPool.py index 5e95005e..5e71155d 100644 --- a/src/Test/TestThreadPool.py +++ b/src/Test/TestThreadPool.py @@ -15,11 +15,9 @@ class TestThreadPool: @pool.wrap def blocker(): events.append("S") - out = 0 - for i in range(10000000): - if i == 3000000: - events.append("M") - out += 1 + time.sleep(0.001) + events.append("M") + time.sleep(0.001) events.append("D") return out @@ -30,9 +28,6 @@ class TestThreadPool: assert events == ["S"] * 3 + ["M"] * 3 + ["D"] * 3 - res = blocker() - assert res == 10000000 - def testLockBlockingSameThread(self): lock = ThreadPool.Lock() From ebd81dad9d99745e903ed905644101684a16e682 Mon Sep 17 00:00:00 2001 From: slrslr <6596726+slrslr@users.noreply.github.com> Date: Sun, 30 Jul 2023 05:17:15 +0000 Subject: [PATCH 03/12] Update README.md Adding new simplified installation commands. It should allow easier installation and update on most Linux distributions using 2 commands. --- README.md | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 0716c869..0a1f641f 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,14 @@ [по-русски](README-ru.md) | [em português](README-ptbr.md) | [简体中文](README-zh-cn.md) -zeronet-conservancy is a fork/continuation of [ZeroNet](https://github.com/HelloZeroNet/ZeroNet) project +Zeronet-Conservancy is a fork/continuation of [ZeroNet](https://github.com/HelloZeroNet/ZeroNet) project (that has been abandoned by its creator) that is dedicated to sustaining existing p2p network and developing its values of decentralization and freedom, while gradually switching to a better designed network ## No active maintainer warning This fork was created and maintained by @caryoscelus, but due to vanishing interest and in order to avoid having -another one-person project, they stepped down. This means there currently is no active maintainer (you're are -welcome to become one!), however some development might still happen. +another one-person project, the development is limitted. ## Why fork? @@ -75,25 +74,34 @@ Following links relate to original ZeroNet: ### Install from your distribution repository -- NixOS: https://search.nixos.org/packages?channel=22.05&show=zeronet-conservancy&type=packages&query=zeronet-conservancy (and see below) +- NixOS: [zeronet-conservancy packages search](https://search.nixos.org/packages?from=0&size=50&sort=relevance&type=packages&query=zeronet-conservancy) (and see below) - ArchLinux: [latest release](https://aur.archlinux.org/packages/zeronet-conservancy), [fresh git version](https://aur.archlinux.org/packages/zeronet-conservancy-git) ### Install from Nix package manager (Linux or MacOS) -``` -# install & configure nix package manager -nix-env -iA nixpkgs.zeronet-conservancy -``` - +if you're on NixOS, install & configure nix package manager: +```nix-env -iA nixpkgs.zeronet-conservancy``` or - -`nix-env -iA nixos.zeronet-conservancy` - -if you're on NixOS - +```nix-env -iA nixos.zeronet-conservancy``` (thanks @fgaz for making & maintaining the package) -### Install from source +### Install from source - simplified, using Git and Python PIP + +1. Install Git and Python PIP package: + +- Debian and Ubuntu based: `sudo apt install git python3-pip -y` +- Red Hat and Fedora based: `yum install epel-release -y 2>/dev/null;yum install git python3 python3-wheel` +- Fedora based dandified: `sudo dnf install git python3-pip python3-wheel -y` +- Arch and Manjaro based: `sudo pacman -S git python-pip -v --no-confirm` +- openSUSE: `sudo zypper install python3-pip python3-setuptools python3-wheel` + +2. Clone Github repository and install required Python modules. First edit zndir path at the begining of the command, to be the path where you want to store Zeronet-Conservancy: + +`zndir="/home/user/myapps/zeronet" ; if [[ ! -d "$zndir" ]]; then git clone --recursive "https://github.com/zeronet-conservancy/zeronet-conservancy.git" "$zndir" && cd "$zndir"||exit; else cd "$zndir";git pull origin master; fi; cd "$zndir" && pip install -r requirements.txt|grep -v "already satisfied"; echo "Try to run: python3 $(pwd)/zeronet.py"` + +(This command can also be used to keep Zeronet-Conservancy up to date) + +### Install from source - detailed #### System dependencies @@ -143,9 +151,9 @@ Install autoconf and other basic development tools, python3 and pip, then procee - after installing general dependencies and cloning repo (as above), run `start-venv.sh` which will create a virtual env for you and install python requirements - more convenience scripts to be added soon -### Building under windows os +### Building under Windows OS -(this instruction is work-in-progress, please help us test it and improve it!) +(These instructions are work-in-progress, please help us test it and improve it!) - install python from https://www.python.org/downloads/ - install some windows compiler suitable for python , this proved to be the most difficult part for me as non-windows user (see here https://wiki.python.org/moin/WindowsCompilers and i'll link more references later) From 30db5a4652dd65f7de47871dd41161d45de11c34 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 20 Oct 2021 19:01:55 +0700 Subject: [PATCH 04/12] Fix https://github.com/HelloZeroNet/ZeroNet/issues/2757 --- src/util/SafeRe.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/util/SafeRe.py b/src/util/SafeRe.py index 6018e2d3..20827f38 100644 --- a/src/util/SafeRe.py +++ b/src/util/SafeRe.py @@ -15,9 +15,10 @@ def isSafePattern(pattern): if unsafe_pattern_match: raise UnsafePatternError("Potentially unsafe part of the pattern: %s in %s" % (unsafe_pattern_match.group(0), pattern)) - repetitions = re.findall(r"\.[\*\{\+]", pattern) - if len(repetitions) >= 10: - raise UnsafePatternError("More than 10 repetitions of %s in %s" % (repetitions[0], pattern)) + repetitions1 = re.findall(r"\.[\*\{\+]", pattern) + repetitions2 = re.findall(r"[^(][?]", pattern) + if len(repetitions1) + len(repetitions2) >= 10: + raise UnsafePatternError("More than 10 repetitions in %s" % pattern) return True From 5fadd5f9bda78e307e82e52e4f889c2ff755e950 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 31 Jul 2023 08:28:29 +0000 Subject: [PATCH 05/12] Improve SafeRe code readability function isSafePattern was never used as boolean function, its only useful behaviour being raising exception on bad regexp, so it's renamed and reused accordingly --- plugins/Sidebar/ConsolePlugin.py | 5 ++--- src/util/SafeRe.py | 12 ++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/plugins/Sidebar/ConsolePlugin.py b/plugins/Sidebar/ConsolePlugin.py index 15f6a1ba..12d49fbf 100644 --- a/plugins/Sidebar/ConsolePlugin.py +++ b/plugins/Sidebar/ConsolePlugin.py @@ -14,8 +14,7 @@ class WsLogStreamer(logging.StreamHandler): self.ui_websocket = ui_websocket if filter: - if not SafeRe.isSafePattern(filter): - raise Exception("Not a safe prex pattern") + SafeRe.guard(filter): self.filter_re = re.compile(".*" + filter) else: self.filter_re = None @@ -55,7 +54,7 @@ class UiWebsocketPlugin(object): pos_start = log_file.tell() lines = [] if filter: - assert SafeRe.isSafePattern(filter) + SafeRe.guard(filter) filter_re = re.compile(".*" + filter) last_match = False diff --git a/src/util/SafeRe.py b/src/util/SafeRe.py index 20827f38..30aa1f29 100644 --- a/src/util/SafeRe.py +++ b/src/util/SafeRe.py @@ -7,7 +7,8 @@ class UnsafePatternError(Exception): cached_patterns = {} -def isSafePattern(pattern): +def guard(pattern): + '''Checks if pattern is safe and raises exception if it isn't''' if len(pattern) > 255: raise UnsafePatternError("Pattern too long: %s characters in %s" % (len(pattern), pattern)) @@ -20,14 +21,13 @@ def isSafePattern(pattern): if len(repetitions1) + len(repetitions2) >= 10: raise UnsafePatternError("More than 10 repetitions in %s" % pattern) - return True - def match(pattern, *args, **kwargs): + '''Guard for safety, compile, cache and match regexp''' cached_pattern = cached_patterns.get(pattern) if cached_pattern: return cached_pattern.match(*args, **kwargs) else: - if isSafePattern(pattern): - cached_patterns[pattern] = re.compile(pattern) - return cached_patterns[pattern].match(*args, **kwargs) + guard(pattern) + cached_patterns[pattern] = re.compile(pattern) + return cached_patterns[pattern].match(*args, **kwargs) From 7d4e66e07b6ced6bfc9c13cf3e9296c21ac517ef Mon Sep 17 00:00:00 2001 From: slrslr <6596726+slrslr@users.noreply.github.com> Date: Mon, 31 Jul 2023 18:34:11 +0000 Subject: [PATCH 06/12] Update README.md Mention first Windows build available for download. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 0a1f641f..1319b13a 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,10 @@ Install autoconf and other basic development tools, python3 and pip, then procee - after installing general dependencies and cloning repo (as above), run `start-venv.sh` which will create a virtual env for you and install python requirements - more convenience scripts to be added soon +### Windows build + +Download and extract .zip archive [zeronet-conservancy-0.7.10-unofficial-win64.zip](https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/zeronet-conservancy-0.7.10-unofficial-win64.zip) + ### Building under Windows OS (These instructions are work-in-progress, please help us test it and improve it!) From b95b979c7fc31a337fc3ca545bab51a0576a1227 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 10 Aug 2023 20:30:24 +0000 Subject: [PATCH 07/12] README: partially rewrite installing --- README.md | 64 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1319b13a..00ce2fda 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [по-русски](README-ru.md) | [em português](README-ptbr.md) | [简体中文](README-zh-cn.md) -Zeronet-Conservancy is a fork/continuation of [ZeroNet](https://github.com/HelloZeroNet/ZeroNet) project +`zeronet-conservancy` is a fork/continuation of [ZeroNet](https://github.com/HelloZeroNet/ZeroNet) project (that has been abandoned by its creator) that is dedicated to sustaining existing p2p network and developing its values of decentralization and freedom, while gradually switching to a better designed network @@ -79,40 +79,39 @@ Following links relate to original ZeroNet: ### Install from Nix package manager (Linux or MacOS) -if you're on NixOS, install & configure nix package manager: -```nix-env -iA nixpkgs.zeronet-conservancy``` -or -```nix-env -iA nixos.zeronet-conservancy``` + - install & configure nix package manager (if needed) + - `nix-env -iA nixpkgs.zeronet-conservancy` + +or add `zeronet-conservancy` to your system configuration if you're on NixOS + (thanks @fgaz for making & maintaining the package) -### Install from source - simplified, using Git and Python PIP - -1. Install Git and Python PIP package: - -- Debian and Ubuntu based: `sudo apt install git python3-pip -y` -- Red Hat and Fedora based: `yum install epel-release -y 2>/dev/null;yum install git python3 python3-wheel` -- Fedora based dandified: `sudo dnf install git python3-pip python3-wheel -y` -- Arch and Manjaro based: `sudo pacman -S git python-pip -v --no-confirm` -- openSUSE: `sudo zypper install python3-pip python3-setuptools python3-wheel` - -2. Clone Github repository and install required Python modules. First edit zndir path at the begining of the command, to be the path where you want to store Zeronet-Conservancy: - -`zndir="/home/user/myapps/zeronet" ; if [[ ! -d "$zndir" ]]; then git clone --recursive "https://github.com/zeronet-conservancy/zeronet-conservancy.git" "$zndir" && cd "$zndir"||exit; else cd "$zndir";git pull origin master; fi; cd "$zndir" && pip install -r requirements.txt|grep -v "already satisfied"; echo "Try to run: python3 $(pwd)/zeronet.py"` - -(This command can also be used to keep Zeronet-Conservancy up to date) - -### Install from source - detailed +### Install from source #### System dependencies ##### Generic unix-like (including mac os x) Install autoconf and other basic development tools, python3 and pip, then proceed to "building python dependencies" +(if running fails due to missing dependency, please report it/make pull request to fix dependency list) ##### Apt-based (debian, ubuntu, etc) - `sudo apt update` - `sudo apt install pkg-config libffi-dev python3-pip python3-venv python3-dev build-essential` +##### Red Hat and Fedora based + - `yum install epel-release -y 2>/dev/null` + - `yum install git python3 python3-wheel` + +##### Fedora based dandified + - `sudo dnf install git python3-pip python3-wheel -y` + +##### openSUSE + - `sudo zypper install python3-pip python3-setuptools python3-wheel` + +##### Arch and Manjaro based + - `sudo pacman -S git python-pip -v --no-confirm` + ##### Android/Termux - install [Termux](https://termux.com/) (in Termux you can install packages via `pkg install `) - `pkg update` @@ -122,7 +121,7 @@ Install autoconf and other basic development tools, python3 and pip, then procee - (optional) `pkg install tor` - (optional) run tor via `tor --ControlPort 9051 --CookieAuthentication 1` command (you can then open new session by swiping to the right) -#### Building python dependencies venv & running +#### Building python dependencies, venv & running - clone this repo (NOTE: on Android/Termux you should clone it into "home" folder of Termux, because virtual environment cannot live in `storage/`) - `python3 -m venv venv` (make python virtual environment, the last `venv` is just a name, if you use different you should replace it in later commands) - `source venv/bin/activate` (activate environment) @@ -147,13 +146,26 @@ Install autoconf and other basic development tools, python3 and pip, then procee - or: `docker compose up -d 0net-tor` for run 0net and tor in one container. (please check if these instructions are still accurate) +#### Alternative one-liner (installing python dependencies globally) + +Clone Github repository and install required Python modules. First +edit zndir path at the begining of the command, to be the path where +you want to store `zeronet-conservancy`: + +`zndir="/home/user/myapps/zeronet" ; if [[ ! -d "$zndir" ]]; then git clone --recursive "https://github.com/zeronet-conservancy/zeronet-conservancy.git" "$zndir" && cd "$zndir"||exit; else cd "$zndir";git pull origin master; fi; cd "$zndir" && pip install -r requirements.txt|grep -v "already satisfied"; echo "Try to run: python3 $(pwd)/zeronet.py"` + +(This command can also be used to keep `zeronet-conservancy` up to date) + #### Alternative script - - after installing general dependencies and cloning repo (as above), run `start-venv.sh` which will create a virtual env for you and install python requirements + - after installing general dependencies and cloning repo (as above), + run `start-venv.sh` which will create a virtual env for you and + install python requirements - more convenience scripts to be added soon -### Windows build +### (unofficial) Windows OS build -Download and extract .zip archive [zeronet-conservancy-0.7.10-unofficial-win64.zip](https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/zeronet-conservancy-0.7.10-unofficial-win64.zip) +Download and extract .zip archive +[zeronet-conservancy-0.7.10-unofficial-win64.zip](https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/zeronet-conservancy-0.7.10-unofficial-win64.zip) ### Building under Windows OS From 21eb421a8f995c5ad6f75f5354e376832efc714d Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 4 Sep 2023 10:26:37 +0000 Subject: [PATCH 08/12] Update data-default --- bootstrap.url | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.url b/bootstrap.url index 8b287997..ec7a3944 100644 --- a/bootstrap.url +++ b/bootstrap.url @@ -1 +1 @@ -https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/data-default-2023-07-26.zip \ No newline at end of file +https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.10/data-default-2023-09-03.zip \ No newline at end of file From 598b8222554812d64e12ab93d4a013f1e5ff9b5c Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 4 Sep 2023 21:25:03 +0000 Subject: [PATCH 09/12] Fix typo in Sidebar/ConsolePlugin --- plugins/Sidebar/ConsolePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/Sidebar/ConsolePlugin.py b/plugins/Sidebar/ConsolePlugin.py index 12d49fbf..46cd9aca 100644 --- a/plugins/Sidebar/ConsolePlugin.py +++ b/plugins/Sidebar/ConsolePlugin.py @@ -14,7 +14,7 @@ class WsLogStreamer(logging.StreamHandler): self.ui_websocket = ui_websocket if filter: - SafeRe.guard(filter): + SafeRe.guard(filter) self.filter_re = re.compile(".*" + filter) else: self.filter_re = None From 337f98a479aae4f8ab1424edf209a94274c125ec Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Mon, 4 Sep 2023 21:39:46 +0000 Subject: [PATCH 10/12] Add --disable_port_check option --- src/Config.py | 1 + src/File/FileServer.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Config.py b/src/Config.py index e92c653c..1f62b184 100644 --- a/src/Config.py +++ b/src/Config.py @@ -256,6 +256,7 @@ class Config(object): self.parser.add_argument('--ip_local', help='My local ips', default=ip_local, type=int, metavar='ip', nargs='*') self.parser.add_argument('--ip_external', help='Set reported external ip (tested on start if None)', metavar='ip', nargs='*') self.parser.add_argument('--offline', help='Disable network communication', action='store_true') + self.parser.add_argument('--disable_port_check', help='Disable checking port', action='store_true') self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true') self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port') diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 41f76817..d1de4761 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -152,9 +152,13 @@ class FileServer(ConnectionServer): FileRequest = imp.load_source("FileRequest", "src/File/FileRequest.py").FileRequest def portCheck(self): - if config.offline or config.tor == 'always': - msg = "Offline mode" if config.offline else "Tor-only" - self.log.info(f'{msg}: port check disabled') + if config.offline or config.tor == 'always' or config.disable_port_check: + if config.offline: + self.log.info(f'Offline mode: port check disabled') + elif config.tor == 'always': + self.log.info('Tor-only mode: port check disabled') + else: + self.log.info('Port check disabled') res = {"ipv4": None, "ipv6": None} self.port_opened = res return res From 77a70e513dffaabbca10cd0659f28dac3d599e2f Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Thu, 7 Sep 2023 20:09:33 +0000 Subject: [PATCH 11/12] README: attribution --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 00ce2fda..59fa72f0 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ Install autoconf and other basic development tools, python3 and pip, then procee - or: `docker compose up -d 0net-tor` for run 0net and tor in one container. (please check if these instructions are still accurate) -#### Alternative one-liner (installing python dependencies globally) +#### Alternative one-liner (by @ssdifnskdjfnsdjk) (installing python dependencies globally) Clone Github repository and install required Python modules. First edit zndir path at the begining of the command, to be the path where From 537d7337e2aa0dcd9a5917e6c341a18d2b9c2895 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Fri, 15 Sep 2023 14:23:19 +0000 Subject: [PATCH 12/12] debug log --- src/Worker/Worker.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Worker/Worker.py b/src/Worker/Worker.py index b7111ba1..9adba30d 100644 --- a/src/Worker/Worker.py +++ b/src/Worker/Worker.py @@ -7,6 +7,8 @@ from Debug import Debug from Config import config from Content.ContentManager import VerifyError +import traceback + class WorkerDownloadError(Exception): pass @@ -119,13 +121,15 @@ class Worker(object): self.manager.log.error("%s: Error writing: %s (%s: %s)" % (self.key, task["inner_path"], type(err), err)) raise WorkerIOError(str(err)) - def onTaskVerifyFail(self, task, error_message): + def onTaskVerifyFail(self, task, error): self.num_failed += 1 if self.manager.started_task_num < 50 or config.verbose: - self.manager.log.debug( + self.manager.log.info( "%s: Verify failed: %s, error: %s, failed peers: %s, workers: %s" % - (self.key, task["inner_path"], error_message, len(task["failed"]), task["workers_num"]) + (self.key, task["inner_path"], error, len(task["failed"]), task["workers_num"]) ) + # traceback.format_ + self.manager.log.debug(''.join(traceback.format_exception(error))) task["failed"].append(self.peer) self.peer.hash_failed += 1 if self.peer.hash_failed >= max(len(self.manager.tasks), 3) or self.peer.connection_error > 10: