New command line action: importBundle; load bootstrap bundle on first run

- importBundle imports zip archive of sites
- latest bootstrapping bundle is downloaded if data/ dir is empty; primary
  reason for this for now is to avoid constantly updating tracker list in
  git tree and use Syncronite instead

requirements.txt

@@ -14,3 +14,4 @@ maxminddb
 rich
 defusedxml>=0.7
 pyaes
+requests

src/Config.py

@@ -320,6 +320,10 @@ class Config(object):
         action.add_argument('cmd', help='API command name')
         action.add_argument('parameters', help='Parameters of the command', nargs='?')
 
+        # Import bundled sites
+        action = self.subparsers.add_parser("importBundle", help='Import sites from a .zip bundle')
+        action.add_argument('bundle', help='Path to a data bundle')
+
         # dbRebuild
         action = self.subparsers.add_parser("dbRebuild", help='Rebuild site database cache')
         action.add_argument('address', help='Site to rebuild')
@@ -426,6 +430,8 @@ class Config(object):
         self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true')
         self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port')
         self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip')
+        self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/caryoscelus/zeronet-conservancy/bootstrap/bootstrap.url', type=str)
+        self.parser.add_argument('--disable_bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true')
         self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*')
         self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', metavar='path', nargs='*')
         self.parser.add_argument('--trackers_proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable")

src/main.py

@@ -34,24 +34,72 @@ def load_config():
 
 load_config()
 
-def init_dirs():
+def importBundle(bundle):
-    if not os.path.isdir(config.data_dir):
+    from zipfile import ZipFile
-        os.mkdir(config.data_dir)
+    from Crypt.CryptBitcoin import isValidAddress
-        try:
+    from Site import SiteManager
-            os.chmod(config.data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+    SiteManager.site_manager.load()
-        except Exception as err:
-            startupError("Can't change permission of %s: %s" % (config.data_dir, err))
 
-    sites_json = f"{config.data_dir}/sites.json"
+    with ZipFile(bundle) as zf:
+        all_files = zf.namelist()
+        top_files = list(set(map(lambda f: f.split('/')[0], all_files)))
+        if len(top_files) == 1 and not isValidAddress(top_files[0]):
+            prefix = top_files[0]+'/'
+        else:
+            prefix = ''
+        top_2 = list(set(filter(lambda f: len(f)>0,
+                                map(lambda f: f.removeprefix(prefix).split('/')[0], all_files))))
+        for d in top_2:
+            if isValidAddress(d):
+                logging.info(f'unpack {d} into {config.data_dir}')
+                for fname in filter(lambda f: f.startswith(prefix+d) and not f.endswith('/'), all_files):
+                    tgt = config.data_dir + '/' + fname.removeprefix(prefix)
+                    logging.info(f'-- {fname} --> {tgt}')
+                    info = zf.getinfo(fname)
+                    info.filename = tgt
+                    zf.extract(info)
+                logging.info(f'add site {d}')
+                SiteManager.site_manager.add(d, noload=True)
+            else:
+                logging.info(f'Warning: unknown file in a bundle: {prefix+d}')
+    SiteManager.site_manager.save()
+
+def init_dirs():
+    data_dir = config.data_dir
+    if not os.path.isdir(data_dir):
+        os.mkdir(data_dir)
+        try:
+            os.chmod(data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+        except Exception as err:
+            startupError(f"Can't change permission of {data_dir}: {err}")
+
+        # download latest bootstrap bundle
+        if not config.disable_bootstrap and not config.offline:
+            import requests
+            from io import BytesIO
+
+            print(f'fetching {config.bootstrap_url}')
+            response = requests.get(config.bootstrap_url)
+            if response.status_code != 200:
+                startupError(f"Cannot load bootstrap bundle (response status: {response.status_code})")
+            url = response.text
+            print(f'got {url}')
+            response = requests.get(url)
+            if response.status_code < 200 or response.status_code >= 300:
+                startupError(f"Cannot load boostrap bundle (response status: {response.status_code})")
+            importBundle(BytesIO(response.content))
+
+    sites_json = f"{data_dir}/sites.json"
     if not os.path.isfile(sites_json):
         with open(sites_json, "w") as f:
             f.write("{}")
-    users_json = f"{config.data_dir}/users.json"
+    users_json = f"{data_dir}/users.json"
     if not os.path.isfile(users_json):
         with open(users_json, "w") as f:
             f.write("{}")
 
 # TODO: GET RID OF TOP-LEVEL CODE!!!
+config.initConsoleLogger()
 
 try:
     init_dirs()
@@ -73,7 +121,7 @@ if config.action == "main":
         r = proc.wait()
         sys.exit(r)
 
-config.initLogging()
+config.initLogging(console_logging=False)
 
 # Debug dependent configuration
 from Debug import DebugHook
@@ -414,6 +462,9 @@ class Actions(object):
         else:
             return res
 
+    def importBundle(self, bundle):
+        importBundle(bundle)
+
     def getWebsocket(self, site):
         import websocket