From bdddf58712abfb03b8ff98abb37351f317ed1ce0 Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Tue, 7 May 2024 14:03:44 +0000 Subject: [PATCH] WIP: change default data directories, subdirectories and config file --- CHANGELOG.md | 5 +- build.py | 2 + plugins/AnnounceShare/AnnounceSharePlugin.py | 2 +- .../AnnounceShare/Test/TestAnnounceShare.py | 2 +- plugins/Chart/ChartDb.py | 2 +- plugins/ContentFilter/ContentFilterStorage.py | 10 +- plugins/FilePack/FilePackPlugin.py | 2 +- plugins/Sidebar/SidebarPlugin.py | 2 +- .../disabled-Bootstrapper/BootstrapperDb.py | 2 +- plugins/disabled-Multiuser/MultiuserPlugin.py | 2 +- .../disabled-Multiuser/Test/TestMultiuser.py | 5 +- src/Actions.py | 4 +- src/Config.py | 247 +++++++++++++----- src/Content/ContentDb.py | 2 +- src/Crypt/CryptConnection.py | 18 +- src/Debug/DebugReloader.py | 2 +- src/Plugin/PluginManager.py | 2 +- src/Site/Site.py | 5 +- src/Site/SiteManager.py | 6 +- src/Site/SiteStorage.py | 2 +- src/Ui/UiRequest.py | 2 +- src/User/User.py | 2 +- src/User/UserManager.py | 2 +- src/main.py | 24 +- src/util/compat.py | 6 + src/util/helper.py | 22 +- zeronet.py | 2 +- 27 files changed, 263 insertions(+), 121 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 172f5f7b..4d9ec20c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ ### zeronet-conservancy 0.7.10+ - disable site-plugins installed for security reasons (@caryoscelus) - fix downloading geoip db (@caryoscelus) -- python <3.6 is officially unsupported +- python <3.6 is officially unsupported (3.8 is more likely minimum requirement) - SafeRe improvements by @geekless - remove and don't update muted files (@caryoscelus) - option to disable port checking (@caryoscelus) @@ -10,6 +10,9 @@ - fix chromium compatibility (@caryoscelus) - better fix of local sites leak (@caryoscelus) - ipython-based repl via --repl for debug/interactive development (@caryoscelus) +- changes in directory structure (split data and config, use user directories by default) +- use version information from git if available +- different build types (portable vs package) - various improvements ### zeronet-conservancy 0.7.10 (2023-07-26) (18d35d3bed4f0683e99) diff --git a/build.py b/build.py index 7b71daf1..e142816a 100755 --- a/build.py +++ b/build.py @@ -32,6 +32,7 @@ def write_to(args, target): f"branch = {branch!r}", f"commit = {commit!r}", f"version = {args.version!r}", + f"platform = {args.platform!r}", ])) def main(): @@ -40,6 +41,7 @@ def main(): parser.add_argument('--version') parser.add_argument('--branch') parser.add_argument('--commit') + parser.add_argument('--platform', default='source') parser.add_argument('--stdout', action=argparse.BooleanOptionalAction, default=False) args = parser.parse_args() if args.stdout: diff --git a/plugins/AnnounceShare/AnnounceSharePlugin.py b/plugins/AnnounceShare/AnnounceSharePlugin.py index b350cf42..2a8a3891 100644 --- a/plugins/AnnounceShare/AnnounceSharePlugin.py +++ b/plugins/AnnounceShare/AnnounceSharePlugin.py @@ -14,7 +14,7 @@ from util import helper class TrackerStorage(object): def __init__(self): self.log = logging.getLogger("TrackerStorage") - self.file_path = "%s/trackers.json" % config.data_dir + self.file_path = config.start_dir / 'trackers.json' self.load() self.time_discover = 0.0 atexit.register(self.save) diff --git a/plugins/AnnounceShare/Test/TestAnnounceShare.py b/plugins/AnnounceShare/Test/TestAnnounceShare.py index 7178eac8..5b820f9b 100644 --- a/plugins/AnnounceShare/Test/TestAnnounceShare.py +++ b/plugins/AnnounceShare/Test/TestAnnounceShare.py @@ -9,7 +9,7 @@ from Config import config @pytest.mark.usefixtures("resetTempSettings") class TestAnnounceShare: def testAnnounceList(self, file_server): - open("%s/trackers.json" % config.data_dir, "w").write("{}") + (config.start_dir / 'trackers.json').open('w').write('{}') tracker_storage = AnnounceSharePlugin.tracker_storage tracker_storage.load() peer = Peer(file_server.ip, 1544, connection_server=file_server) diff --git a/plugins/Chart/ChartDb.py b/plugins/Chart/ChartDb.py index 66a22082..3bb449e8 100644 --- a/plugins/Chart/ChartDb.py +++ b/plugins/Chart/ChartDb.py @@ -6,7 +6,7 @@ import time class ChartDb(Db): def __init__(self): self.version = 2 - super(ChartDb, self).__init__(self.getSchema(), "%s/chart.db" % config.data_dir) + super(ChartDb, self).__init__(self.getSchema(), config.start_dir / 'chart.db') self.foreign_keys = True self.checkTables() self.sites = self.loadSites() diff --git a/plugins/ContentFilter/ContentFilterStorage.py b/plugins/ContentFilter/ContentFilterStorage.py index 2ad378d6..7d62e7e4 100644 --- a/plugins/ContentFilter/ContentFilterStorage.py +++ b/plugins/ContentFilter/ContentFilterStorage.py @@ -14,7 +14,7 @@ from util import helper class ContentFilterStorage(object): def __init__(self, site_manager): self.log = logging.getLogger("ContentFilterStorage") - self.file_path = "%s/filters.json" % config.data_dir + self.file_path = config.config_dir / 'filters.json' self.site_manager = site_manager self.file_content = self.load() @@ -36,12 +36,12 @@ class ContentFilterStorage(object): def load(self): # Rename previously used mutes.json -> filters.json - if os.path.isfile("%s/mutes.json" % config.data_dir): + if (config.config_dir / 'mutes.json').is_file(): self.log.info("Renaming mutes.json to filters.json...") - os.rename("%s/mutes.json" % config.data_dir, self.file_path) - if os.path.isfile(self.file_path): + os.rename(config.config_dir / 'mutes.json', self.file_path) + if self.file_path.is_file(): try: - return json.load(open(self.file_path)) + return json.load(self.file_path.open()) except Exception as err: self.log.error("Error loading filters.json: %s" % err) return None diff --git a/plugins/FilePack/FilePackPlugin.py b/plugins/FilePack/FilePackPlugin.py index 1c931316..488ff1a0 100644 --- a/plugins/FilePack/FilePackPlugin.py +++ b/plugins/FilePack/FilePackPlugin.py @@ -44,7 +44,7 @@ class UiRequestPlugin(object): if ".zip/" in path or ".tar.gz/" in path: file_obj = None path_parts = self.parsePath(path) - file_path = "%s/%s/%s" % (config.data_dir, path_parts["address"], path_parts["inner_path"]) + file_path = config.data_dir / path_parts["address"] / path_parts["inner_path"] match = re.match(r"^(.*\.(?:tar.gz|zip))/(.*)", file_path) archive_path, path_within = match.groups() if archive_path not in archive_cache: diff --git a/plugins/Sidebar/SidebarPlugin.py b/plugins/Sidebar/SidebarPlugin.py index b8c5f0f3..c24a0e0b 100644 --- a/plugins/Sidebar/SidebarPlugin.py +++ b/plugins/Sidebar/SidebarPlugin.py @@ -686,7 +686,7 @@ class UiWebsocketPlugin(object): if sys.platform == "linux": sys_db_paths += ['/usr/share/GeoIP/' + db_name] - data_dir_db_path = os.path.join(config.data_dir, db_name) + data_dir_db_path = config.data_dir / db_name db_paths = sys_db_paths + [data_dir_db_path] diff --git a/plugins/disabled-Bootstrapper/BootstrapperDb.py b/plugins/disabled-Bootstrapper/BootstrapperDb.py index 0866dc3e..355fac8c 100644 --- a/plugins/disabled-Bootstrapper/BootstrapperDb.py +++ b/plugins/disabled-Bootstrapper/BootstrapperDb.py @@ -12,7 +12,7 @@ class BootstrapperDb(Db.Db): def __init__(self): self.version = 7 self.hash_ids = {} # hash -> id cache - super(BootstrapperDb, self).__init__({"db_name": "Bootstrapper"}, "%s/bootstrapper.db" % config.data_dir) + super(BootstrapperDb, self).__init__({"db_name": "Bootstrapper"}, config.start_dir / 'bootstrapper.db') self.foreign_keys = True self.checkTables() self.updateHashCache() diff --git a/plugins/disabled-Multiuser/MultiuserPlugin.py b/plugins/disabled-Multiuser/MultiuserPlugin.py index a2fd79ae..342307bf 100644 --- a/plugins/disabled-Multiuser/MultiuserPlugin.py +++ b/plugins/disabled-Multiuser/MultiuserPlugin.py @@ -16,7 +16,7 @@ def importPluginnedClasses(): from User import UserManager try: - local_master_addresses = set(json.load(open("%s/users.json" % config.data_dir)).keys()) # Users in users.json + local_master_addresses = set(json.load((config.private_dir / 'users.json').open()).keys()) # Users in users.json except Exception as err: local_master_addresses = set() diff --git a/plugins/disabled-Multiuser/Test/TestMultiuser.py b/plugins/disabled-Multiuser/Test/TestMultiuser.py index b8ff4267..fe03833d 100644 --- a/plugins/disabled-Multiuser/Test/TestMultiuser.py +++ b/plugins/disabled-Multiuser/Test/TestMultiuser.py @@ -8,7 +8,8 @@ from User import UserManager class TestMultiuser: def testMemorySave(self, user): # It should not write users to disk - users_before = open("%s/users.json" % config.data_dir).read() + users_json = config.private_dir / 'users.json' + users_before = users_json.open().read() user = UserManager.user_manager.create() user.save() - assert open("%s/users.json" % config.data_dir).read() == users_before + assert users_json.open().read() == users_before diff --git a/src/Actions.py b/src/Actions.py index 7a1fe7d9..72c8b063 100644 --- a/src/Actions.py +++ b/src/Actions.py @@ -92,8 +92,8 @@ class Actions: from Site import SiteManager SiteManager.site_manager.load() - os.mkdir("%s/%s" % (config.data_dir, address)) - open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address) + (config.data_dir / address).mkdir() + (config.data_dir / address / 'index.html').open('w').write(f"Hello {address}!") logging.info("Creating content.json...") site = Site(address) diff --git a/src/Config.py b/src/Config.py index 12603d40..2895f5fd 100644 --- a/src/Config.py +++ b/src/Config.py @@ -9,10 +9,18 @@ import logging import logging.handlers import stat import time +from pathlib import Path VERSION = "0.7.10+" +class StartupError(RuntimeError): + pass + class Config: + """Class responsible for storing and loading config. + + Used as singleton `config` + """ def __init__(self, argv): try: @@ -23,11 +31,13 @@ class Config: self.branch = Git.branch() or 'unknown' self.commit = Git.commit() or 'unknown' self.version = VERSION + self.platform = 'source' else: self.build_type = Build.build_type self.branch = Build.branch self.commit = Build.commit self.version = Build.version or VERSION + self.platform = Build.platform self.version_full = f'{self.version} ({self.build_type} from {self.branch}-{self.commit})' self.user_agent = "conservancy" # for compatibility @@ -45,15 +55,18 @@ class Config: self.keys_restart_need = set([ "tor", "fileserver_port", "fileserver_ip_type", "threads_fs_read", "threads_fs_write", "threads_crypt", "threads_db" ]) - self.start_dir = self.getStartDir() - self.config_file = self.start_dir + "/zeronet.conf" - self.private_dir = self.start_dir + '/private' - self.data_dir = self.start_dir + "/data" - self.log_dir = self.start_dir + "/log" + + self.config_file = None + self.config_dir = None + self.data_dir = None + self.private_dir = None + self.log_dir = None + self.configurePaths(argv) + self.openssl_lib_file = None self.openssl_bin_file = None - self.trackers_file = False + self.trackers_file = None self.createParser() self.createArguments() @@ -70,46 +83,21 @@ class Config: def strToBool(self, v): return v.lower() in ("yes", "true", "t", "1") - def getStartDir(self): - """Return directory with config & data""" + def getStartDirOld(self): + """Get directory that would have been used by older versions (pre v0.7.11)""" + this_file = os.path.abspath(__file__).replace("\\", "/").rstrip("cd") + if "--start-dir" in self.argv: - return self.argv[self.argv.index("--start-dir") + 1] - - if '--portable' in self.argv or self.build_type == 'portable': - return '.' - - here = os.path.dirname(os.path.abspath(__file__).replace("\\", "/")).rstrip('/src') - if os.path.isdir(f'{here}/data') and not '--no-portable' in self.argv: - print('WARNING: found data in current directory') - print(' It used to be default behaviour to store data alongside project directory,') - print(' but now we default to place data and config in user home directory.') - print(' If you want to keep previous behaviour, please use --portable') - print('Assuming implicit --portable (use --no-portable to override)') - print(self.argv) - self.argv.insert(1, '--portable') - print(self.argv) - return '.' - - home_zn = os.path.expanduser(f'~/ZeroNet') - if os.path.isdir(home_zn): - print(f'WARNING: found data in {home_zn}') - print( ' It is possible that this is from previous version or another installation') - print( ' altogether. If you want to use that data directory with zeronet-conservancy') - print(f' you have to run it with --start-dir "{home_zn}" option') - - if platform.system() == 'Linux': - # TODO: XDG! - return os.path.expanduser('~/.local/zeronet-conservancy') - - if platform.system() == 'Darwin': - return os.path.expanduser("~/Library/Application Support/zeronet-conservancy") - - if platform.system() == 'Windows': - return os.path.expanduser('~/AppData/zeronet-conservancy') - - elif here.endswith("/Contents/Resources/core/src"): - start_dir = os.path.expanduser("~/Library/Application Support/ZeroNet") - elif this_file.endswith("/core/src"): + start_dir = self.argv[self.argv.index("--start-dir") + 1] + elif this_file.endswith("/Contents/Resources/core/src/Config.py"): + # Running as ZeroNet.app + if this_file.startswith("/Application") or this_file.startswith("/private") or this_file.startswith(os.path.expanduser("~/Library")): + # Runnig from non-writeable directory, put data to Application Support + start_dir = os.path.expanduser("~/Library/Application Support/ZeroNet") + else: + # Running from writeable directory put data next to .app + start_dir = re.sub("/[^/]+/Contents/Resources/core/src/Config.py", "", this_file) + elif this_file.endswith("/core/src/Config.py"): # Running as exe or source is at Application Support directory, put var files to outside of core dir start_dir = this_file.replace("/core/src/Config.py", "") elif not os.access(this_file.replace('/src/Config.py', ''), os.R_OK | os.W_OK): @@ -117,6 +105,144 @@ class Config: start_dir = os.path.expanduser("~/ZeroNet") else: start_dir = "." + return start_dir + + def migrateOld(self, source): + print(f'[bold red]WARNING: found data {source}[/bold red]') + print( ' It used to be default behaviour to store data there,') + print( ' but now we default to place data and config in user home directory.') + print( '') + + def configurePaths(self, argv): + if '--config-file' in argv: + self.config_file = argv[argv.index('--config-file') + 1] + old_dir = Path(self.getStartDirOld()) + new_dir = Path(self.getStartDir()) + no_migrate = '--no-migrate' in argv + silent_migrate = '--portable' in argv or '--migrate' in argv + try: + self.start_dir = self.maybeMigrate(old_dir, new_dir, no_migrate, silent_migrate) + except Exception as ex: + raise ex + + self.updatePaths() + + def updatePaths(self): + if self.config_file is None: + self.config_file = self.start_dir / 'znc.conf' + if self.config_dir is None: + self.config_dir = self.start_dir + if self.private_dir is None: + self.private_dir = self.start_dir / 'private' + if self.data_dir is None: + self.data_dir = self.start_dir / 'data' + if self.log_dir is None: + self.log_dir = self.start_dir / 'log' + + def createPaths(self): + self.start_dir.mkdir(parents=True, exist_ok=True) + self.private_dir.mkdir(parents=True, exist_ok=True) + self.data_dir.mkdir(parents=True, exist_ok=True) + self.log_dir.mkdir(parents=True, exist_ok=True) + + def checkDir(self, root): + return (root / 'znc.conf').is_file() + + def doMigrate(self, old_dir, new_dir): + raise RuntimeError('Migration not implemented yet') + + def askMigrate(self, old_dir, new_dir, silent): + if not sys.stdin.isatty(): + raise StartupError('Migration refused: non-interactive shell') + while True: + r = input(f'You have old data in `{old_dir}`. Migrate to new format to `{new_dir}`? [Y/n]') + if r.lower().startswith('n'): + raise StartupError('Migration refused') + if r.lower().startswith('y'): + return self.doMigrate(old_dir, new_dir) + + def createNewConfig(self, new_dir): + new_dir.mkdir(parents=True, exist_ok=True) + with (new_dir / 'znc.conf').open('w') as f: + f.write('# zeronet-conervancy config file') + + def maybeMigrate(self, old_dir, new_dir, no_migrate, silent_migrate): + if old_dir.exists() and new_dir.exists(): + if old_dir == new_dir: + if self.checkDir(new_dir): + return new_dir + elif no_migrate: + return StartError('Migration refused, but new directory should be migrated') + else: + return askMigrate(old_dir, new_dir, silent_migrate) + else: + if self.checkDir(new_dir): + if not no_migrate: + print("There's an old starting directory") + return new_dir + else: + raise StartupError('Bad startup directory') + elif old_dir.exists(): + if no_migrate: + self.createNewConfig(new_dir) + return new_dir + else: + return self.askMigrate(old_dir, new_dir, silent_migrate) + elif new_dir.exists(): + if self.checkDir(new_dir): + return new_dir + else: + return StartupError('Bad startup directory') + else: + self.createNewConfig(new_dir) + return new_dir + + def getStartDir(self): + """Return directory with config & data""" + if "--start-dir" in self.argv: + return self.argv[self.argv.index("--start-dir") + 1] + + here = os.path.dirname(os.path.abspath(__file__).replace("\\", "/")).rstrip('/src') + if '--portable' in self.argv or self.build_type == 'portable': + return here + + # if os.path.isdir(f'{here}/data') and not '--no-portable' in self.argv: + # print('WARNING: found data in current directory') + # print(' It used to be default behaviour to store data alongside project directory,') + # print(' but now we default to place data and config in user home directory.') + # print(' If you want to keep previous behaviour, please use --portable') + # print('Assuming implicit --portable (use --no-portable to override)') + # print(self.argv) + # self.argv.insert(1, '--portable') + # print(self.argv) + # return here + + # home_zn = os.path.expanduser(f'~/ZeroNet') + # if os.path.isdir(home_zn): + # print(f'WARNING: found data in {home_zn}') + # print( ' It is possible that this is from previous version or another installation') + # print( ' altogether. If you want to use that data directory with zeronet-conservancy') + # print(f' you have to run it with --start-dir "{home_zn}" option') + + MACOSX_DIR = '~/Library/Application Support/zeronet-conservancy' + WINDOWS_DIR = '~/AppData/zeronet-conservancy' + LIBREDESKTOP_DIR = '~/.local/share/zeronet-conservancy' + if self.platform == 'source': + if platform.system() == 'Darwin': + path = MACOSX_DIR + elif platform.system() == 'Windows': + path = WINDOWS_DIR + else: + path = LIBREDESKTOP_DIR + elif self.platform == 'macosx': + path = MACOSX_DIR + elif self.platform == 'windows': + path = WINDOWS_DIR + elif self.platform == 'libredesktop': + path = LIBREDESKTOP_DIR + else: + raise RuntimeError(f'UNKNOWN PLATFORM: {self.platform}. Something must have went terribly wrong!') + return os.path.expanduser(path) # Create command line arguments def createArguments(self): @@ -135,9 +261,9 @@ class Config: else: fix_float_decimals = False - config_file = self.start_dir + "/zeronet.conf" - data_dir = self.start_dir + "/data" - log_dir = self.start_dir + "/log" + config_file = self.config_file + data_dir = self.data_dir + log_dir = self.log_dir ip_local = ["127.0.0.1", "::1"] @@ -256,9 +382,10 @@ class Config: self.parser.add_argument('--batch', help="Batch mode (No interactive input for commands)", action='store_true') self.parser.add_argument('--portable', action=argparse.BooleanOptionalAction) - self.parser.add_argument('--start-dir', help='Path of working dir for variable content (data, log, .conf)', default=self.start_dir, metavar="path") + self.parser.add_argument('--start-dir', help='Path of working dir for variable content (data, log, config)', default=self.start_dir, metavar="path") self.parser.add_argument('--config-file', help='Path of config file', default=config_file, metavar="path") self.parser.add_argument('--data-dir', help='Path of data directory', default=data_dir, metavar="path") + self.parser.add_argument('--no-migrate', help='Ignore data directories from old 0net versions', action=argparse.BooleanOptionalAction, default=False) self.parser.add_argument('--console-log-level', help='Level of logging to console', default="default", choices=["default", "DEBUG", "INFO", "ERROR", "off"]) @@ -352,7 +479,7 @@ class Config: return self.parser def loadTrackersFile(self): - if not self.trackers_file: + if self.trackers_file is None: return None self.trackers = self.arguments.trackers[:] @@ -362,16 +489,19 @@ class Config: if trackers_file.startswith("/"): # Absolute trackers_file_path = trackers_file elif trackers_file.startswith("{data_dir}"): # Relative to data_dir - trackers_file_path = trackers_file.replace("{data_dir}", self.data_dir) - else: # Relative to zeronet.py - trackers_file_path = self.start_dir + "/" + trackers_file + trackers_file_path = trackers_file.replace('{data_dir}', str(self.data_dir)) + else: + # Relative to zeronet.py or something else, unsupported + raise RuntimeError(f'trackers_file should be relative to {{data_dir}} or absolute path (not {trackers_file})') for line in open(trackers_file_path): tracker = line.strip() if "://" in tracker and tracker not in self.trackers: self.trackers.append(tracker) except Exception as err: - print("Error loading trackers file: %s" % err) + print(self.trackers_file) + print(trackers_file) + print(f'Error loading trackers file: {err}') # Find arguments specified for current action def getActionArguments(self): @@ -436,6 +566,8 @@ class Config: self.parseCommandline(argv, silent) # Parse argv self.setAttributes() + self.updatePaths() + self.createPaths() if parse_config: argv = self.parseConfig(argv) # Add arguments from config file @@ -460,7 +592,7 @@ class Config: for arg in args: if arg.startswith('--') and '_' in arg: farg = arg.replace('_', '-') - print(f'WARNING: using deprecated flag in command line: {arg} should be {farg}') + print(f'[bold red]WARNING: using deprecated flag in command line: {arg} should be {farg}[/bold red]') print('Support for deprecated flags might be removed in the future') else: farg = arg @@ -494,9 +626,6 @@ class Config: def parseConfig(self, argv): argv = self.fixArgs(argv) - # Find config file path from parameters - if "--config-file" in argv: - self.config_file = argv[argv.index("--config-file") + 1] # Load config file if os.path.isfile(self.config_file): config = configparser.RawConfigParser(allow_no_value=True, strict=False) @@ -539,7 +668,7 @@ class Config: val = val[:] if key in ("data_dir", "log_dir", "start_dir", "openssl_bin_file", "openssl_lib_file"): if val: - val = val.replace("\\", "/") + val = Path(val) setattr(self, key, val) def loadPlugins(self): diff --git a/src/Content/ContentDb.py b/src/Content/ContentDb.py index f284581e..5b63993d 100644 --- a/src/Content/ContentDb.py +++ b/src/Content/ContentDb.py @@ -153,7 +153,7 @@ content_dbs = {} def getContentDb(path=None): if not path: - path = "%s/content.db" % config.data_dir + path = config.data_dir / 'content.db' if path not in content_dbs: content_dbs[path] = ContentDb(path) content_dbs[path].init() diff --git a/src/Crypt/CryptConnection.py b/src/Crypt/CryptConnection.py index ebbc6295..c7f3ea5b 100644 --- a/src/Crypt/CryptConnection.py +++ b/src/Crypt/CryptConnection.py @@ -24,20 +24,20 @@ class CryptConnectionManager: self.context_server = None self.openssl_conf_template = "src/lib/openssl/openssl.cnf" - self.openssl_conf = config.data_dir + "/openssl.cnf" + self.openssl_conf = config.private_dir / "openssl.cnf" self.openssl_env = { "OPENSSL_CONF": self.openssl_conf, - "RANDFILE": config.data_dir + "/openssl-rand.tmp" + "RANDFILE": config.private_dir / "openssl-rand.tmp" } self.crypt_supported = [] # Supported cryptos - self.cacert_pem = config.data_dir + "/cacert-rsa.pem" - self.cakey_pem = config.data_dir + "/cakey-rsa.pem" - self.cert_pem = config.data_dir + "/cert-rsa.pem" - self.cert_csr = config.data_dir + "/cert-rsa.csr" - self.key_pem = config.data_dir + "/key-rsa.pem" + self.cacert_pem = config.private_dir / "cacert-rsa.pem" + self.cakey_pem = config.private_dir / "cakey-rsa.pem" + self.cert_pem = config.private_dir / "cert-rsa.pem" + self.cert_csr = config.private_dir / "cert-rsa.csr" + self.key_pem = config.private_dir / "key-rsa.pem" self.log = logging.getLogger("CryptConnectionManager") self.log.debug("Version: %s" % ssl.OPENSSL_VERSION) @@ -105,8 +105,8 @@ class CryptConnectionManager: if config.keep_ssl_cert: return False for file_name in ["cert-rsa.pem", "key-rsa.pem", "cacert-rsa.pem", "cakey-rsa.pem", "cacert-rsa.srl", "cert-rsa.csr", "openssl-rand.tmp"]: - file_path = "%s/%s" % (config.data_dir, file_name) - if os.path.isfile(file_path): + file_path = config.data_dir / file_name + if file_path.is_file(): os.unlink(file_path) # Load and create cert files is necessary diff --git a/src/Debug/DebugReloader.py b/src/Debug/DebugReloader.py index 482c7921..be9b4d8c 100644 --- a/src/Debug/DebugReloader.py +++ b/src/Debug/DebugReloader.py @@ -21,7 +21,7 @@ else: class DebugReloader: def __init__(self, paths=None): if not paths: - paths = ["src", "plugins", config.data_dir + "/__plugins__"] + paths = ["src", "plugins"] self.log = logging.getLogger("DebugReloader") self.last_chaged = 0 self.callbacks = [] diff --git a/src/Plugin/PluginManager.py b/src/Plugin/PluginManager.py index ab0940e8..82db4cfd 100644 --- a/src/Plugin/PluginManager.py +++ b/src/Plugin/PluginManager.py @@ -25,7 +25,7 @@ class PluginManager: self.after_load = [] # Execute functions after loaded plugins self.function_flags = {} # Flag function for permissions self.reloading = False - self.config_path = config.data_dir + "/plugins.json" + self.config_path = config.config_dir / 'plugins.json' self.loadConfig() self.config.setdefault("builtin", {}) diff --git a/src/Site/Site.py b/src/Site/Site.py index ad0e3ca2..becc7d3e 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -88,9 +88,10 @@ class Site(object): def loadSettings(self, settings=None): if not settings: try: - settings = json.load(open(f'{config.data_dir}/sites.json')).get(self.address) + with (config.private_dir / 'sites.json').open() as f: + settings = json.load(f).get(self.address) except Exception as err: - logging.error(f'Error loading {config.data_dir}/sites.json: {err}') + logging.error(f'Error loading {config.private_dir}/sites.json: {err}') settings = {} if settings: self.settings = settings diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 78c20f86..0fb3b060 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -38,7 +38,7 @@ class SiteManager(object): load_s = time.time() # Load new adresses try: - json_path = f"{config.data_dir}/sites.json" + json_path = config.private_dir / 'sites.json' data = json.load(open(json_path)) except Exception as err: self.log.error(f"Unable to load {json_path}: {err}") @@ -48,7 +48,7 @@ class SiteManager(object): for address, settings in data.items(): if address not in self.sites: - if os.path.isfile("%s/%s/content.json" % (config.data_dir, address)): + if (config.data_dir / address / 'content.json').is_file(): # Root content.json exists, try load site s = time.time() try: @@ -121,7 +121,7 @@ class SiteManager(object): s = time.time() if data: - helper.atomicWrite("%s/sites.json" % config.data_dir, helper.jsonDumps(data).encode("utf8")) + helper.atomicWrite(config.private_dir / 'sites.json', helper.jsonDumps(data).encode("utf8")) else: self.log.debug("Save error: No data") time_write = time.time() - s diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index be8f88e9..7249ad34 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -29,7 +29,7 @@ thread_pool_fs_batch = ThreadPool.ThreadPool(1, name="FS batch") class SiteStorage(object): def __init__(self, site, allow_create=True): self.site = site - self.directory = f'{config.data_dir}/{self.site.address}' # Site data diretory + self.directory = config.data_dir / self.site.address # Site data diretory self.allowed_dir = os.path.abspath(self.directory) # Only serve file within this dir self.log = site.log self.db = None # Db class diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 34af96e2..967508da 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -784,7 +784,7 @@ class UiRequest: address = path_parts["address"] - file_path = "%s/%s/%s" % (config.data_dir, address, path_parts["inner_path"]) + file_path = config.data_dir / address / path_parts['inner_path'] if (config.debug or config.merge_media) and file_path.split("/")[-1].startswith("all."): # If debugging merge *.css to all.css and *.js to all.js diff --git a/src/User/User.py b/src/User/User.py index 89571146..bbf18f07 100644 --- a/src/User/User.py +++ b/src/User/User.py @@ -35,7 +35,7 @@ class User(object): # Save to data/users.json @util.Noparallel(queue=True, ignore_class=True) def save(self): - users_json = f'{config.private_dir}/users.json' + users_json = config.private_dir / 'users.json' s = time.time() users = json.load(open(users_json)) if self.master_address not in users: diff --git a/src/User/UserManager.py b/src/User/UserManager.py index a2afc295..b8e49664 100644 --- a/src/User/UserManager.py +++ b/src/User/UserManager.py @@ -25,7 +25,7 @@ class UserManager(object): s = time.time() # Load new users try: - json_path = f'{config.private_dir}/users.json' + json_path = config.private_dir / 'users.json' data = json.load(open(json_path)) except Exception as err: raise Exception("Unable to load %s: %s" % (json_path, err)) diff --git a/src/main.py b/src/main.py index 2fff005f..a6622432 100644 --- a/src/main.py +++ b/src/main.py @@ -34,7 +34,7 @@ def importBundle(bundle): from Crypt.CryptBitcoin import isValidAddress import json - sites_json_path = f"{config.data_dir}/sites.json" + sites_json_path = config.private_dir / 'sites.json' try: with open(sites_json_path) as f: sites = json.load(f) @@ -68,16 +68,17 @@ def importBundle(bundle): def init_dirs(): data_dir = Path(config.data_dir) + private_dir = Path(config.private_dir) need_bootstrap = (config.bootstrap and not config.offline - and (not data_dir.is_dir() or not (data_dir / 'sites.json').is_file())) + and (not data_dir.is_dir() or not (private_dir / 'sites.json').is_file())) - old_users_json = data_dir / 'users.json' - if old_users_json.is_file(): - print('Migrating existing users.json file to private/') - old_sites_json = data_dir / 'sites.json' - if old_sites_json.is_file(): - print('Migrating existing sites.json file to private/') + # old_users_json = data_dir / 'users.json' + # if old_users_json.is_file(): + # print('Migrating existing users.json file to private/') + # old_sites_json = data_dir / 'sites.json' + # if old_sites_json.is_file(): + # print('Migrating existing sites.json file to private/') if not data_dir.is_dir(): data_dir.mkdir(parents=True, exist_ok=True) @@ -97,11 +98,11 @@ def init_dirs(): startupError(f"Cannot load boostrap bundle (response status: {response.status_code})") importBundle(BytesIO(response.content)) - sites_json = f"{data_dir}/sites.json" + sites_json = private_dir / 'sites.json' if not os.path.isfile(sites_json): with open(sites_json, "w") as f: f.write("{}") - users_json = f"{private_dir_dir}/users.json" + users_json = private_dir / 'users.json' if not os.path.isfile(users_json): with open(users_json, "w") as f: f.write("{}") @@ -117,7 +118,6 @@ def init(): config.initConsoleLogger() try: - print(config.start_dir) init_dirs() except: import traceback as tb @@ -129,7 +129,7 @@ def init(): if config.action == "main": from util import helper try: - lock = helper.openLocked(f"{config.data_dir}/lock.pid", "w") + lock = helper.openLocked(config.data_dir / 'lock.pid', "w") lock.write(f"{os.getpid()}") except BlockingIOError as err: startupError(f"Can't open lock file, your 0net client is probably already running, exiting... ({err})") diff --git a/src/util/compat.py b/src/util/compat.py index f41e67b2..1867dad0 100644 --- a/src/util/compat.py +++ b/src/util/compat.py @@ -14,3 +14,9 @@ else: return s.removeprefix(prefix) def removesuffix(s, suffix, /): return s.removesuffix(suffix) + +import argparse + +if not hasattr(argparse, 'BooleanOptionalAction'): + from .argparseCompat import BooleanOptionalAction + argparse.BooleanOptionalAction = BooleanOptionalAction diff --git a/src/util/helper.py b/src/util/helper.py index af65f727..62264c89 100644 --- a/src/util/helper.py +++ b/src/util/helper.py @@ -16,17 +16,17 @@ from Config import config def atomicWrite(dest, content, mode="wb"): try: - with open(dest + "-tmpnew", mode) as f: + with open(f'{dest}-tmpnew', mode) as f: f.write(content) f.flush() os.fsync(f.fileno()) - if os.path.isfile(dest + "-tmpold"): # Previous incomplete write - os.rename(dest + "-tmpold", dest + "-tmpold-%s" % time.time()) + if os.path.isfile(f'{dest}-tmpold'): # Previous incomplete write + os.rename(f'{dest}-tmpold', f'{dest}-tmpold-{time.time()}') if os.path.isfile(dest): # Rename old file to -tmpold - os.rename(dest, dest + "-tmpold") - os.rename(dest + "-tmpnew", dest) - if os.path.isfile(dest + "-tmpold"): - os.unlink(dest + "-tmpold") # Remove old file + os.rename(dest, f'{dest}-tmpold') + os.rename(f'{dest}-tmpnew', dest) + if os.path.isfile(f'{dest}-tmpold'): + os.unlink(f'{dest}-tmpold') # Remove old file return True except Exception as err: from Debug import Debug @@ -34,8 +34,8 @@ def atomicWrite(dest, content, mode="wb"): "File %s write failed: %s, (%s) reverting..." % (dest, Debug.formatException(err), Debug.formatStack()) ) - if os.path.isfile(dest + "-tmpold") and not os.path.isfile(dest): - os.rename(dest + "-tmpold", dest) + if os.path.isfile(f'{dest}-tmpold') and not os.path.isfile(dest): + os.rename(f'{dest}-tmpold', dest) return False @@ -85,7 +85,7 @@ def openLocked(path, mode="wb"): def getFreeSpace(): free_space = -1 if "statvfs" in dir(os): # Unix - statvfs = os.statvfs(config.data_dir.encode("utf8")) + statvfs = os.statvfs(str(config.data_dir).encode("utf8")) free_space = statvfs.f_frsize * statvfs.f_bavail else: # Windows try: @@ -111,7 +111,7 @@ def shellquote(*args): if len(args) == 1: return '"%s"' % args[0].replace('"', "") else: - return tuple(['"%s"' % arg.replace('"', "") for arg in args]) + return tuple(['"%s"' % str(arg).replace('"', "") for arg in args]) def packPeers(peers): diff --git a/zeronet.py b/zeronet.py index ff752fb6..4931b4f7 100755 --- a/zeronet.py +++ b/zeronet.py @@ -34,7 +34,7 @@ def launch(): except Exception as log_err: print("Failed to log error:", log_err) traceback.print_exc() - error_log_path = config.log_dir + "/error.log" + error_log_path = config.log_dir / "error.log" traceback.print_exc(file=open(error_log_path, "w")) print("---") print("Please report it: https://github.com/zeronet-conservancy/zeronet-conservancy/issues/new?template=bug-report.md")