diff --git a/bootstrap.url b/bootstrap.url new file mode 100644 index 00000000..f23462eb --- /dev/null +++ b/bootstrap.url @@ -0,0 +1 @@ +https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.9/data-default-2023-07-23.zip \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4298ed61..887138d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ maxminddb rich defusedxml>=0.7 pyaes +requests diff --git a/src/Config.py b/src/Config.py index fc30a924..28a6e19e 100644 --- a/src/Config.py +++ b/src/Config.py @@ -9,178 +9,6 @@ import logging.handlers import stat import time -trackers = [ - 'zero://188.242.242.224:26474', - 'zero://2001:19f0:8001:1d2f:5400:2ff:fe83:5bf7:23141', - 'zero://200:1e7a:5100:ef7c:6fa4:d8ae:b91c:a74:15441', - 'zero://23.184.48.134:15441', - 'zero://57hzgtu62yzxqgbvgxs7g3lfck3za4zrda7qkskar3tlak5recxcebyd.onion:15445', - 'zero://6i54dd5th73oelv636ivix6sjnwfgk2qsltnyvswagwphub375t3xcad.onion:15441', - 'zero://f2hnjbggc3c2u2apvxdugirnk6bral54ibdoul3hhvu7pd4fso5fq3yd.onion:15441', - 'zero://gugt43coc5tkyrhrc3esf6t6aeycvcqzw7qafxrjpqbwt4ssz5czgzyd.onion:15441', - 'zero://k5w77dozo3hy5zualyhni6vrh73iwfkaofa64abbilwyhhd3wgenbjqd.onion:15441', - 'zero://ow7in4ftwsix5klcbdfqvfqjvimqshbm2o75rhtpdnsderrcbx74wbad.onion:15441', - 'zero://pn4q2zzt2pw4nk7yidxvsxmydko7dfibuzxdswi6gu6ninjpofvqs2id.onion:15441', - 'zero://skdeywpgm5xncpxbbr4cuiip6ey4dkambpanog6nruvmef4f3e7o47qd.onion:15441', - 'zero://wlxav3szbrdhest4j7dib2vgbrd7uj7u7rnuzg22cxbih7yxyg2hsmid.onion:15441', - 'zero://zy7wttvjtsijt5uwmlar4yguvjc2gppzbdj4v6bujng6xwjmkdg7uvqd.onion:15441', - 'http://bt.okmp3.ru:2710/announce', - 'http://fxtt.ru:80/announce', - 'http://incine.ru:6969/announce', - 'http://moeweb.pw:6969/announce', - 'http://open.acgnxtracker.com:80/announce', - 'http://t.acg.rip:6699/announce', - 'http://t.nyaatracker.com:80/announce', - 'http://t.overflow.biz:6969/announce', - 'http://tracker.files.fm:6969/announce', - 'http://tracker.mywaifu.best:6969/announce', - 'http://tracker.vrpnet.org:6969/announce', - 'http://vps02.net.orel.ru:80/announce', - 'udp://960303.xyz:6969/announce', - 'udp://aarsen.me:6969/announce', - 'udp://astrr.ru:6969/announce', - 'udp://ben.kerbertools.xyz:6969/announce', - 'udp://bt1.archive.org:6969/announce', - 'udp://bt2.archive.org:6969/announce', - 'udp://bt.ktrackers.com:6666/announce', - 'udp://bubu.mapfactor.com:6969/announce', - 'udp://c.ns.cluefone.com:6969/announce', - 'udp://cutscloud.duckdns.org:6969/announce', - 'udp://download.nerocloud.me:6969/announce', - 'udp://epider.me:6969/announce', - 'udp://exodus.desync.com:6969/announce', - 'udp://htz3.noho.st:6969/announce', - 'udp://ipv4.tracker.harry.lu:80/announce', - 'udp://laze.cc:6969/announce', - 'udp://mail.artixlinux.org:6969/announce', - 'udp://mirror.aptus.co.tz:6969/announce', - 'udp://moonburrow.club:6969/announce', - 'udp://movies.zsw.ca:6969/announce', - 'udp://mts.tvbit.co:6969/announce', - 'udp://new-line.net:6969/announce', - 'udp://open.demonii.com:1337/announce', - 'udp://open.stealth.si:80/announce', - 'udp://opentracker.i2p.rocks:6969/announce', - 'udp://p4p.arenabg.com:1337/announce', - 'udp://psyco.fr:6969/announce', - 'udp://public.publictracker.xyz:6969/announce', - 'udp://rep-art.ynh.fr:6969/announce', - 'udp://run.publictracker.xyz:6969/announce', - 'udp://sanincode.com:6969/announce', - 'udp://slicie.icon256.com:8000/announce', - 'udp://tamas3.ynh.fr:6969/announce', - 'udp://thouvenin.cloud:6969/announce', - 'udp://torrentclub.space:6969/announce', - 'udp://tracker.0x.tf:6969/announce', - 'udp://tracker1.bt.moack.co.kr:80/announce', - 'udp://tracker.4.babico.name.tr:3131/announce', - 'udp://tracker.altrosky.nl:6969/announce', - 'udp://tracker.artixlinux.org:6969/announce', - 'udp://tracker.farted.net:6969/announce', - 'udp://tracker.jonaslsa.com:6969/announce', - 'udp://tracker.joybomb.tw:6969/announce', - 'udp://tracker.monitorit4.me:6969/announce', - 'udp://tracker.opentrackr.org:1337/announce', - 'udp://tracker.pomf.se:80/announce', - 'udp://tracker.publictracker.xyz:6969/announce', - 'udp://tracker.srv00.com:6969/announce', - 'udp://tracker.tcp.exchange:6969/announce', - 'udp://tracker.theoks.net:6969/announce', - 'udp://transkaroo.joustasie.net:6969/announce', - 'udp://uploads.gamecoast.net:6969/announce', - 'udp://v2.iperson.xyz:6969/announce', - 'udp://vibe.sleepyinternetfun.xyz:1738/announce', - 'udp://www.skynetcenter.me:6969/announce', - 'udp://www.torrent.eu.org:451/announce', - 'zero://194.5.98.39:15441', - 'zero://145.239.95.38:15441', - 'zero://178.128.34.249:26117', - 'zero://217.18.217.143:39288', - 'zero://83.246.141.203:22207', - 'zero://syncronite.loki:15441', - 'zero://2a05:dfc1:4000:1e00::a:15441', - 'zero://2400:6180:100:d0::8fd:8001:21697', - 'zero://2001:19f0:8001:1d2f:5400:2ff:fe83:5bf7:30530', - 'zero://73pyhfwfwsrhfw76knkjfnw6o3lk53zfo7hlxdmxbj75sjcnol5cioad.onion:15442', - 'zero://fzlzmxuz2bust72cuy5g4w6d62tx624xcjaupf2kp7ffuitbiniy2hqd.onion:15441', - 'zero://rlcjomszyitxpwv7kzopmqgzk3bdpsxeull4c3s6goszkk6h2sotfoad.onion:15441', - 'zero://tqmo2nffqo4qc5jgmz3me5eri3zpgf3v2zciufzmhnvznjve5c3argad.onion:15441', - 'http://107.189.31.134:6969/announce', - 'http://119.28.71.45:8080/announce', - 'http://129.146.193.240:6699/announce', - 'http://159.69.65.157:6969/announce', - 'http://163.172.29.130:80/announce', - 'http://185.130.47.2:6969/announce', - 'http://45.67.35.111:6969/announce', - 'http://61.222.178.254:6969/announce', - 'http://83.31.30.182:6969/announce', - 'http://93.158.213.92:1337/announce', - 'http://95.217.167.10:6969/announce', - 'udp://102.223.180.235:6969/announce', - 'udp://103.122.21.50:6969/announce', - 'udp://104.131.98.232:6969/announce', - 'udp://104.244.77.87:6969/announce', - 'udp://107.189.11.58:6969/announce', - 'udp://107.189.31.134:6969/announce', - 'udp://139.144.68.88:6969/announce', - 'udp://149.28.239.70:6969/announce', - 'udp://15.204.205.14:6969/announce', - 'udp://156.234.201.18:80/announce', - 'udp://158.101.161.60:3131/announce', - 'udp://163.172.29.130:80/announce', - 'udp://167.99.185.219:6969/announce', - 'udp://176.31.250.174:6969/announce', - 'udp://176.56.4.238:6969/announce', - 'udp://178.32.222.98:3391/announce', - 'udp://184.105.151.166:6969/announce', - 'udp://185.102.219.163:6969/announce', - 'udp://185.181.60.155:80/announce', - 'udp://185.217.199.21:6969/announce', - 'udp://185.44.82.25:1337/announce', - 'udp://185.68.21.244:6969/announce', - 'udp://192.3.165.191:6969/announce', - 'udp://192.3.165.198:6969/announce', - 'udp://192.95.46.115:6969/announce', - 'udp://193.176.158.162:6969/announce', - 'udp://193.37.214.12:6969/announce', - 'udp://193.42.111.57:9337/announce', - 'udp://198.100.149.66:6969/announce', - 'udp://20.100.205.229:6969/announce', - 'udp://207.241.226.111:6969/announce', - 'udp://207.241.231.226:6969/announce', - 'udp://209.141.59.16:6969/announce', - 'udp://212.237.53.230:6969/announce', - 'udp://23.153.248.2:6969/announce', - 'udp://23.254.228.89:6969/announce', - 'udp://37.187.111.136:6969/announce', - 'udp://37.27.4.53:6969/announce', - 'udp://38.7.201.142:6969/announce', - 'udp://45.154.253.6:6969/announce', - 'udp://45.63.30.114:6969/announce', - 'udp://45.9.60.30:6969/announce', - 'udp://46.38.238.105:6969/announce', - 'udp://49.12.76.8:8080/announce', - 'udp://5.102.159.190:6969/announce', - 'udp://5.196.89.204:6969/announce', - 'udp://51.15.79.209:6969/announce', - 'udp://51.159.54.68:6666/announce', - 'udp://51.68.174.87:6969/announce', - 'udp://51.81.222.188:6969/announce', - 'udp://52.58.128.163:6969/announce', - 'udp://61.222.178.254:6969/announce', - 'udp://77.73.69.230:6969/announce', - 'udp://83.102.180.21:80/announce', - 'udp://83.31.30.182:6969/announce', - 'udp://85.206.172.159:6969/announce', - 'udp://85.239.33.28:6969/announce', - 'udp://86.57.161.157:6969/announce', - 'udp://91.216.110.52:451/announce', - 'udp://93.158.213.92:1337/announce', - 'udp://94.103.87.87:6969/announce', - 'udp://95.216.74.39:6969/announce', - 'udp://95.31.11.224:6969/announce', -] - class Config(object): def __init__(self, argv): @@ -322,6 +150,10 @@ class Config(object): action.add_argument('cmd', help='API command name') action.add_argument('parameters', help='Parameters of the command', nargs='?') + # Import bundled sites + action = self.subparsers.add_parser("importBundle", help='Import sites from a .zip bundle') + action.add_argument('bundle', help='Path to a data bundle') + # dbRebuild action = self.subparsers.add_parser("dbRebuild", help='Rebuild site database cache') action.add_argument('address', help='Site to rebuild') @@ -428,8 +260,10 @@ class Config(object): self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true') self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port') self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip') - self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*') - self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', metavar='path', nargs='*') + self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/caryoscelus/zeronet-conservancy/bootstrap/bootstrap.url', type=str) + self.parser.add_argument('--disable_bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true') + self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=[], metavar='protocol://address', nargs='*') + self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', default=['{data_dir}/15CEFKBRHFfAP9rmL6hhLmHoXrrgmw4B5o/cache/1/Syncronite.html'], metavar='path', nargs='*') self.parser.add_argument('--trackers_proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable") self.parser.add_argument('--use_libsecp256k1', help='Use Libsecp256k1 liblary for speedup', type='bool', choices=[True, False], default=True) self.parser.add_argument('--use_openssl', help='Use OpenSSL liblary for speedup', type='bool', choices=[True, False], default=True) @@ -799,10 +633,6 @@ class Config(object): except Exception as err: print("Can't change permission of %s: %s" % (self.log_dir, err)) - # Make warning hidden from console - logging.WARNING = 15 # Don't display warnings if not in debug mode - logging.addLevelName(15, "WARNING") - logging.getLogger('').name = "-" # Remove root prefix self.error_logger = ErrorLogHandler() diff --git a/src/Crypt/CryptBitcoin.py b/src/Crypt/CryptBitcoin.py index 68b2caa2..a0807187 100644 --- a/src/Crypt/CryptBitcoin.py +++ b/src/Crypt/CryptBitcoin.py @@ -99,3 +99,16 @@ def verify(data, valid_address, sign, lib_verify=None): # Verify data using add return sign_address in valid_address else: # One possible address return sign_address == valid_address + +def isValidAddress(addr): + '''Check if provided address is valid bitcoin address''' + if addr[0] != '1': + # no support for new-style addrs + return False + from base58 import b58decode + bs = b58decode(addr) + main = bs[:-4] + checksum = bs[-4:] + h1 = hashlib.sha256(main).digest() + h2 = hashlib.sha256(h1).digest() + return h2[:4] == checksum diff --git a/src/Site/Site.py b/src/Site/Site.py index ffdb2bb0..ae2be36a 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -87,7 +87,11 @@ class Site(object): # Load site settings from data/sites.json def loadSettings(self, settings=None): if not settings: - settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address) + try: + settings = json.load(open(f'{config.data_dir}/sites.json')).get(self.address) + except Exception as err: + logging.error(f'Error loading {config.data_dir}/sites.json: {err}') + settings = {} if settings: self.settings = settings if "cache" not in settings: diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index 2fd63e82..1c4252ae 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -202,7 +202,7 @@ class SiteAnnouncer(object): else: raise AnnounceError("Unknown protocol: %s" % address_parts["protocol"]) except Exception as err: - self.site.log.warning("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode)) + self.site.log.debug("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode)) error = err if error: diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index d7ba6e94..78c20f86 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -31,17 +31,18 @@ class SiteManager(object): @util.Noparallel() def load(self, cleanup=True, startup=False): from .Site import Site - self.log.info("Loading sites... (cleanup: %s, startup: %s)" % (cleanup, startup)) + self.log.info(f'Loading sites... ({cleanup=}, {startup=})') self.loaded = False address_found = [] added = 0 load_s = time.time() # Load new adresses try: - json_path = "%s/sites.json" % config.data_dir + json_path = f"{config.data_dir}/sites.json" data = json.load(open(json_path)) except Exception as err: - raise Exception("Unable to load %s: %s" % (json_path, err)) + self.log.error(f"Unable to load {json_path}: {err}") + data = {} sites_need = [] diff --git a/src/main.py b/src/main.py index 181d883b..7f5c1b33 100644 --- a/src/main.py +++ b/src/main.py @@ -34,24 +34,79 @@ def load_config(): load_config() -def init_dirs(): - if not os.path.isdir(config.data_dir): - os.mkdir(config.data_dir) - try: - os.chmod(config.data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - except Exception as err: - startupError("Can't change permission of %s: %s" % (config.data_dir, err)) +def importBundle(bundle): + from zipfile import ZipFile + from Crypt.CryptBitcoin import isValidAddress + import json - sites_json = f"{config.data_dir}/sites.json" + sites_json_path = f"{config.data_dir}/sites.json" + try: + with open(sites_json_path) as f: + sites = json.load(f) + except Exception as err: + sites = {} + + with ZipFile(bundle) as zf: + all_files = zf.namelist() + top_files = list(set(map(lambda f: f.split('/')[0], all_files))) + if len(top_files) == 1 and not isValidAddress(top_files[0]): + prefix = top_files[0]+'/' + else: + prefix = '' + top_2 = list(set(filter(lambda f: len(f)>0, + map(lambda f: f.removeprefix(prefix).split('/')[0], all_files)))) + for d in top_2: + if isValidAddress(d): + logging.info(f'unpack {d} into {config.data_dir}') + for fname in filter(lambda f: f.startswith(prefix+d) and not f.endswith('/'), all_files): + tgt = config.data_dir + '/' + fname.removeprefix(prefix) + logging.info(f'-- {fname} --> {tgt}') + info = zf.getinfo(fname) + info.filename = tgt + zf.extract(info) + logging.info(f'add site {d}') + sites[d] = {} + else: + logging.info(f'Warning: unknown file in a bundle: {prefix+d}') + with open(sites_json_path, 'w') as f: + json.dump(sites, f) + +def init_dirs(): + data_dir = config.data_dir + if not os.path.isdir(data_dir): + os.mkdir(data_dir) + try: + os.chmod(data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + except Exception as err: + startupError(f"Can't change permission of {data_dir}: {err}") + + # download latest bootstrap bundle + if not config.disable_bootstrap and not config.offline: + import requests + from io import BytesIO + + print(f'fetching {config.bootstrap_url}') + response = requests.get(config.bootstrap_url) + if response.status_code != 200: + startupError(f"Cannot load bootstrap bundle (response status: {response.status_code})") + url = response.text + print(f'got {url}') + response = requests.get(url) + if response.status_code < 200 or response.status_code >= 300: + startupError(f"Cannot load boostrap bundle (response status: {response.status_code})") + importBundle(BytesIO(response.content)) + + sites_json = f"{data_dir}/sites.json" if not os.path.isfile(sites_json): with open(sites_json, "w") as f: f.write("{}") - users_json = f"{config.data_dir}/users.json" + users_json = f"{data_dir}/users.json" if not os.path.isfile(users_json): with open(users_json, "w") as f: f.write("{}") # TODO: GET RID OF TOP-LEVEL CODE!!! +config.initConsoleLogger() try: init_dirs() @@ -73,7 +128,7 @@ if config.action == "main": r = proc.wait() sys.exit(r) -config.initLogging() +config.initLogging(console_logging=False) # Debug dependent configuration from Debug import DebugHook @@ -415,6 +470,9 @@ class Actions(object): else: return res + def importBundle(self, bundle): + importBundle(bundle) + def getWebsocket(self, site): import websocket