From 9444e097a66f9ab12562a27231fa0ba4420f37ae Mon Sep 17 00:00:00 2001 From: caryoscelus Date: Sun, 23 Jul 2023 21:14:54 +0000 Subject: [PATCH] New command line action: importBundle; load bootstrap bundle on first run - importBundle imports zip archive of sites - latest bootstrapping bundle is downloaded if data/ dir is empty; primary reason for this for now is to avoid constantly updating tracker list in git tree and use Syncronite instead --- requirements.txt | 1 + src/Config.py | 6 ++++ src/main.py | 71 +++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4298ed61..887138d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ maxminddb rich defusedxml>=0.7 pyaes +requests diff --git a/src/Config.py b/src/Config.py index e1c397e2..8850679b 100644 --- a/src/Config.py +++ b/src/Config.py @@ -320,6 +320,10 @@ class Config(object): action.add_argument('cmd', help='API command name') action.add_argument('parameters', help='Parameters of the command', nargs='?') + # Import bundled sites + action = self.subparsers.add_parser("importBundle", help='Import sites from a .zip bundle') + action.add_argument('bundle', help='Path to a data bundle') + # dbRebuild action = self.subparsers.add_parser("dbRebuild", help='Rebuild site database cache') action.add_argument('address', help='Site to rebuild') @@ -426,6 +430,8 @@ class Config(object): self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true') self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port') self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip') + self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/caryoscelus/zeronet-conservancy/bootstrap/bootstrap.url', type=str) + self.parser.add_argument('--disable_bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true') self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*') self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', metavar='path', nargs='*') self.parser.add_argument('--trackers_proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable") diff --git a/src/main.py b/src/main.py index b4b656db..b99e0076 100644 --- a/src/main.py +++ b/src/main.py @@ -34,24 +34,72 @@ def load_config(): load_config() -def init_dirs(): - if not os.path.isdir(config.data_dir): - os.mkdir(config.data_dir) - try: - os.chmod(config.data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - except Exception as err: - startupError("Can't change permission of %s: %s" % (config.data_dir, err)) +def importBundle(bundle): + from zipfile import ZipFile + from Crypt.CryptBitcoin import isValidAddress + from Site import SiteManager + SiteManager.site_manager.load() - sites_json = f"{config.data_dir}/sites.json" + with ZipFile(bundle) as zf: + all_files = zf.namelist() + top_files = list(set(map(lambda f: f.split('/')[0], all_files))) + if len(top_files) == 1 and not isValidAddress(top_files[0]): + prefix = top_files[0]+'/' + else: + prefix = '' + top_2 = list(set(filter(lambda f: len(f)>0, + map(lambda f: f.removeprefix(prefix).split('/')[0], all_files)))) + for d in top_2: + if isValidAddress(d): + logging.info(f'unpack {d} into {config.data_dir}') + for fname in filter(lambda f: f.startswith(prefix+d) and not f.endswith('/'), all_files): + tgt = config.data_dir + '/' + fname.removeprefix(prefix) + logging.info(f'-- {fname} --> {tgt}') + info = zf.getinfo(fname) + info.filename = tgt + zf.extract(info) + logging.info(f'add site {d}') + SiteManager.site_manager.add(d, noload=True) + else: + logging.info(f'Warning: unknown file in a bundle: {prefix+d}') + SiteManager.site_manager.save() + +def init_dirs(): + data_dir = config.data_dir + if not os.path.isdir(data_dir): + os.mkdir(data_dir) + try: + os.chmod(data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + except Exception as err: + startupError(f"Can't change permission of {data_dir}: {err}") + + # download latest bootstrap bundle + if not config.disable_bootstrap and not config.offline: + import requests + from io import BytesIO + + print(f'fetching {config.bootstrap_url}') + response = requests.get(config.bootstrap_url) + if response.status_code != 200: + startupError(f"Cannot load bootstrap bundle (response status: {response.status_code})") + url = response.text + print(f'got {url}') + response = requests.get(url) + if response.status_code < 200 or response.status_code >= 300: + startupError(f"Cannot load boostrap bundle (response status: {response.status_code})") + importBundle(BytesIO(response.content)) + + sites_json = f"{data_dir}/sites.json" if not os.path.isfile(sites_json): with open(sites_json, "w") as f: f.write("{}") - users_json = f"{config.data_dir}/users.json" + users_json = f"{data_dir}/users.json" if not os.path.isfile(users_json): with open(users_json, "w") as f: f.write("{}") # TODO: GET RID OF TOP-LEVEL CODE!!! +config.initConsoleLogger() try: init_dirs() @@ -73,7 +121,7 @@ if config.action == "main": r = proc.wait() sys.exit(r) -config.initLogging() +config.initLogging(console_logging=False) # Debug dependent configuration from Debug import DebugHook @@ -414,6 +462,9 @@ class Actions(object): else: return res + def importBundle(self, bundle): + importBundle(bundle) + def getWebsocket(self, site): import websocket