Merge pull request #219 from caryoscelus/bootstrap
Bootstrap 0net trackers on first launch
This commit is contained in:
caryoscelus
GitHub
commit
615e155043
8 changed files with 101 additions and 193 deletions
1
bootstrap.url
Normal file
1
bootstrap.url
Normal file
|
|
@ -0,0 +1 @@
|
|||
https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.9/data-default-2023-07-23.zip
|
||||
|
|
@ -14,3 +14,4 @@ maxminddb
|
|||
rich
|
||||
defusedxml>=0.7
|
||||
pyaes
|
||||
requests
|
||||
|
|
|
|||
186
src/Config.py
186
src/Config.py
|
|
@ -9,178 +9,6 @@ import logging.handlers
|
|||
import stat
|
||||
import time
|
||||
|
||||
trackers = [
|
||||
'zero://188.242.242.224:26474',
|
||||
'zero://2001:19f0:8001:1d2f:5400:2ff:fe83:5bf7:23141',
|
||||
'zero://200:1e7a:5100:ef7c:6fa4:d8ae:b91c:a74:15441',
|
||||
'zero://23.184.48.134:15441',
|
||||
'zero://57hzgtu62yzxqgbvgxs7g3lfck3za4zrda7qkskar3tlak5recxcebyd.onion:15445',
|
||||
'zero://6i54dd5th73oelv636ivix6sjnwfgk2qsltnyvswagwphub375t3xcad.onion:15441',
|
||||
'zero://f2hnjbggc3c2u2apvxdugirnk6bral54ibdoul3hhvu7pd4fso5fq3yd.onion:15441',
|
||||
'zero://gugt43coc5tkyrhrc3esf6t6aeycvcqzw7qafxrjpqbwt4ssz5czgzyd.onion:15441',
|
||||
'zero://k5w77dozo3hy5zualyhni6vrh73iwfkaofa64abbilwyhhd3wgenbjqd.onion:15441',
|
||||
'zero://ow7in4ftwsix5klcbdfqvfqjvimqshbm2o75rhtpdnsderrcbx74wbad.onion:15441',
|
||||
'zero://pn4q2zzt2pw4nk7yidxvsxmydko7dfibuzxdswi6gu6ninjpofvqs2id.onion:15441',
|
||||
'zero://skdeywpgm5xncpxbbr4cuiip6ey4dkambpanog6nruvmef4f3e7o47qd.onion:15441',
|
||||
'zero://wlxav3szbrdhest4j7dib2vgbrd7uj7u7rnuzg22cxbih7yxyg2hsmid.onion:15441',
|
||||
'zero://zy7wttvjtsijt5uwmlar4yguvjc2gppzbdj4v6bujng6xwjmkdg7uvqd.onion:15441',
|
||||
'http://bt.okmp3.ru:2710/announce',
|
||||
'http://fxtt.ru:80/announce',
|
||||
'http://incine.ru:6969/announce',
|
||||
'http://moeweb.pw:6969/announce',
|
||||
'http://open.acgnxtracker.com:80/announce',
|
||||
'http://t.acg.rip:6699/announce',
|
||||
'http://t.nyaatracker.com:80/announce',
|
||||
'http://t.overflow.biz:6969/announce',
|
||||
'http://tracker.files.fm:6969/announce',
|
||||
'http://tracker.mywaifu.best:6969/announce',
|
||||
'http://tracker.vrpnet.org:6969/announce',
|
||||
'http://vps02.net.orel.ru:80/announce',
|
||||
'udp://960303.xyz:6969/announce',
|
||||
'udp://aarsen.me:6969/announce',
|
||||
'udp://astrr.ru:6969/announce',
|
||||
'udp://ben.kerbertools.xyz:6969/announce',
|
||||
'udp://bt1.archive.org:6969/announce',
|
||||
'udp://bt2.archive.org:6969/announce',
|
||||
'udp://bt.ktrackers.com:6666/announce',
|
||||
'udp://bubu.mapfactor.com:6969/announce',
|
||||
'udp://c.ns.cluefone.com:6969/announce',
|
||||
'udp://cutscloud.duckdns.org:6969/announce',
|
||||
'udp://download.nerocloud.me:6969/announce',
|
||||
'udp://epider.me:6969/announce',
|
||||
'udp://exodus.desync.com:6969/announce',
|
||||
'udp://htz3.noho.st:6969/announce',
|
||||
'udp://ipv4.tracker.harry.lu:80/announce',
|
||||
'udp://laze.cc:6969/announce',
|
||||
'udp://mail.artixlinux.org:6969/announce',
|
||||
'udp://mirror.aptus.co.tz:6969/announce',
|
||||
'udp://moonburrow.club:6969/announce',
|
||||
'udp://movies.zsw.ca:6969/announce',
|
||||
'udp://mts.tvbit.co:6969/announce',
|
||||
'udp://new-line.net:6969/announce',
|
||||
'udp://open.demonii.com:1337/announce',
|
||||
'udp://open.stealth.si:80/announce',
|
||||
'udp://opentracker.i2p.rocks:6969/announce',
|
||||
'udp://p4p.arenabg.com:1337/announce',
|
||||
'udp://psyco.fr:6969/announce',
|
||||
'udp://public.publictracker.xyz:6969/announce',
|
||||
'udp://rep-art.ynh.fr:6969/announce',
|
||||
'udp://run.publictracker.xyz:6969/announce',
|
||||
'udp://sanincode.com:6969/announce',
|
||||
'udp://slicie.icon256.com:8000/announce',
|
||||
'udp://tamas3.ynh.fr:6969/announce',
|
||||
'udp://thouvenin.cloud:6969/announce',
|
||||
'udp://torrentclub.space:6969/announce',
|
||||
'udp://tracker.0x.tf:6969/announce',
|
||||
'udp://tracker1.bt.moack.co.kr:80/announce',
|
||||
'udp://tracker.4.babico.name.tr:3131/announce',
|
||||
'udp://tracker.altrosky.nl:6969/announce',
|
||||
'udp://tracker.artixlinux.org:6969/announce',
|
||||
'udp://tracker.farted.net:6969/announce',
|
||||
'udp://tracker.jonaslsa.com:6969/announce',
|
||||
'udp://tracker.joybomb.tw:6969/announce',
|
||||
'udp://tracker.monitorit4.me:6969/announce',
|
||||
'udp://tracker.opentrackr.org:1337/announce',
|
||||
'udp://tracker.pomf.se:80/announce',
|
||||
'udp://tracker.publictracker.xyz:6969/announce',
|
||||
'udp://tracker.srv00.com:6969/announce',
|
||||
'udp://tracker.tcp.exchange:6969/announce',
|
||||
'udp://tracker.theoks.net:6969/announce',
|
||||
'udp://transkaroo.joustasie.net:6969/announce',
|
||||
'udp://uploads.gamecoast.net:6969/announce',
|
||||
'udp://v2.iperson.xyz:6969/announce',
|
||||
'udp://vibe.sleepyinternetfun.xyz:1738/announce',
|
||||
'udp://www.skynetcenter.me:6969/announce',
|
||||
'udp://www.torrent.eu.org:451/announce',
|
||||
'zero://194.5.98.39:15441',
|
||||
'zero://145.239.95.38:15441',
|
||||
'zero://178.128.34.249:26117',
|
||||
'zero://217.18.217.143:39288',
|
||||
'zero://83.246.141.203:22207',
|
||||
'zero://syncronite.loki:15441',
|
||||
'zero://2a05:dfc1:4000:1e00::a:15441',
|
||||
'zero://2400:6180:100:d0::8fd:8001:21697',
|
||||
'zero://2001:19f0:8001:1d2f:5400:2ff:fe83:5bf7:30530',
|
||||
'zero://73pyhfwfwsrhfw76knkjfnw6o3lk53zfo7hlxdmxbj75sjcnol5cioad.onion:15442',
|
||||
'zero://fzlzmxuz2bust72cuy5g4w6d62tx624xcjaupf2kp7ffuitbiniy2hqd.onion:15441',
|
||||
'zero://rlcjomszyitxpwv7kzopmqgzk3bdpsxeull4c3s6goszkk6h2sotfoad.onion:15441',
|
||||
'zero://tqmo2nffqo4qc5jgmz3me5eri3zpgf3v2zciufzmhnvznjve5c3argad.onion:15441',
|
||||
'http://107.189.31.134:6969/announce',
|
||||
'http://119.28.71.45:8080/announce',
|
||||
'http://129.146.193.240:6699/announce',
|
||||
'http://159.69.65.157:6969/announce',
|
||||
'http://163.172.29.130:80/announce',
|
||||
'http://185.130.47.2:6969/announce',
|
||||
'http://45.67.35.111:6969/announce',
|
||||
'http://61.222.178.254:6969/announce',
|
||||
'http://83.31.30.182:6969/announce',
|
||||
'http://93.158.213.92:1337/announce',
|
||||
'http://95.217.167.10:6969/announce',
|
||||
'udp://102.223.180.235:6969/announce',
|
||||
'udp://103.122.21.50:6969/announce',
|
||||
'udp://104.131.98.232:6969/announce',
|
||||
'udp://104.244.77.87:6969/announce',
|
||||
'udp://107.189.11.58:6969/announce',
|
||||
'udp://107.189.31.134:6969/announce',
|
||||
'udp://139.144.68.88:6969/announce',
|
||||
'udp://149.28.239.70:6969/announce',
|
||||
'udp://15.204.205.14:6969/announce',
|
||||
'udp://156.234.201.18:80/announce',
|
||||
'udp://158.101.161.60:3131/announce',
|
||||
'udp://163.172.29.130:80/announce',
|
||||
'udp://167.99.185.219:6969/announce',
|
||||
'udp://176.31.250.174:6969/announce',
|
||||
'udp://176.56.4.238:6969/announce',
|
||||
'udp://178.32.222.98:3391/announce',
|
||||
'udp://184.105.151.166:6969/announce',
|
||||
'udp://185.102.219.163:6969/announce',
|
||||
'udp://185.181.60.155:80/announce',
|
||||
'udp://185.217.199.21:6969/announce',
|
||||
'udp://185.44.82.25:1337/announce',
|
||||
'udp://185.68.21.244:6969/announce',
|
||||
'udp://192.3.165.191:6969/announce',
|
||||
'udp://192.3.165.198:6969/announce',
|
||||
'udp://192.95.46.115:6969/announce',
|
||||
'udp://193.176.158.162:6969/announce',
|
||||
'udp://193.37.214.12:6969/announce',
|
||||
'udp://193.42.111.57:9337/announce',
|
||||
'udp://198.100.149.66:6969/announce',
|
||||
'udp://20.100.205.229:6969/announce',
|
||||
'udp://207.241.226.111:6969/announce',
|
||||
'udp://207.241.231.226:6969/announce',
|
||||
'udp://209.141.59.16:6969/announce',
|
||||
'udp://212.237.53.230:6969/announce',
|
||||
'udp://23.153.248.2:6969/announce',
|
||||
'udp://23.254.228.89:6969/announce',
|
||||
'udp://37.187.111.136:6969/announce',
|
||||
'udp://37.27.4.53:6969/announce',
|
||||
'udp://38.7.201.142:6969/announce',
|
||||
'udp://45.154.253.6:6969/announce',
|
||||
'udp://45.63.30.114:6969/announce',
|
||||
'udp://45.9.60.30:6969/announce',
|
||||
'udp://46.38.238.105:6969/announce',
|
||||
'udp://49.12.76.8:8080/announce',
|
||||
'udp://5.102.159.190:6969/announce',
|
||||
'udp://5.196.89.204:6969/announce',
|
||||
'udp://51.15.79.209:6969/announce',
|
||||
'udp://51.159.54.68:6666/announce',
|
||||
'udp://51.68.174.87:6969/announce',
|
||||
'udp://51.81.222.188:6969/announce',
|
||||
'udp://52.58.128.163:6969/announce',
|
||||
'udp://61.222.178.254:6969/announce',
|
||||
'udp://77.73.69.230:6969/announce',
|
||||
'udp://83.102.180.21:80/announce',
|
||||
'udp://83.31.30.182:6969/announce',
|
||||
'udp://85.206.172.159:6969/announce',
|
||||
'udp://85.239.33.28:6969/announce',
|
||||
'udp://86.57.161.157:6969/announce',
|
||||
'udp://91.216.110.52:451/announce',
|
||||
'udp://93.158.213.92:1337/announce',
|
||||
'udp://94.103.87.87:6969/announce',
|
||||
'udp://95.216.74.39:6969/announce',
|
||||
'udp://95.31.11.224:6969/announce',
|
||||
]
|
||||
|
||||
class Config(object):
|
||||
|
||||
def __init__(self, argv):
|
||||
|
|
@ -322,6 +150,10 @@ class Config(object):
|
|||
action.add_argument('cmd', help='API command name')
|
||||
action.add_argument('parameters', help='Parameters of the command', nargs='?')
|
||||
|
||||
# Import bundled sites
|
||||
action = self.subparsers.add_parser("importBundle", help='Import sites from a .zip bundle')
|
||||
action.add_argument('bundle', help='Path to a data bundle')
|
||||
|
||||
# dbRebuild
|
||||
action = self.subparsers.add_parser("dbRebuild", help='Rebuild site database cache')
|
||||
action.add_argument('address', help='Site to rebuild')
|
||||
|
|
@ -428,8 +260,10 @@ class Config(object):
|
|||
self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true')
|
||||
self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port')
|
||||
self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip')
|
||||
self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*')
|
||||
self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', metavar='path', nargs='*')
|
||||
self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/caryoscelus/zeronet-conservancy/bootstrap/bootstrap.url', type=str)
|
||||
self.parser.add_argument('--disable_bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true')
|
||||
self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=[], metavar='protocol://address', nargs='*')
|
||||
self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', default=['{data_dir}/15CEFKBRHFfAP9rmL6hhLmHoXrrgmw4B5o/cache/1/Syncronite.html'], metavar='path', nargs='*')
|
||||
self.parser.add_argument('--trackers_proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable")
|
||||
self.parser.add_argument('--use_libsecp256k1', help='Use Libsecp256k1 liblary for speedup', type='bool', choices=[True, False], default=True)
|
||||
self.parser.add_argument('--use_openssl', help='Use OpenSSL liblary for speedup', type='bool', choices=[True, False], default=True)
|
||||
|
|
@ -799,10 +633,6 @@ class Config(object):
|
|||
except Exception as err:
|
||||
print("Can't change permission of %s: %s" % (self.log_dir, err))
|
||||
|
||||
# Make warning hidden from console
|
||||
logging.WARNING = 15 # Don't display warnings if not in debug mode
|
||||
logging.addLevelName(15, "WARNING")
|
||||
|
||||
logging.getLogger('').name = "-" # Remove root prefix
|
||||
|
||||
self.error_logger = ErrorLogHandler()
|
||||
|
|
|
|||
|
|
@ -99,3 +99,16 @@ def verify(data, valid_address, sign, lib_verify=None): # Verify data using add
|
|||
return sign_address in valid_address
|
||||
else: # One possible address
|
||||
return sign_address == valid_address
|
||||
|
||||
def isValidAddress(addr):
|
||||
'''Check if provided address is valid bitcoin address'''
|
||||
if addr[0] != '1':
|
||||
# no support for new-style addrs
|
||||
return False
|
||||
from base58 import b58decode
|
||||
bs = b58decode(addr)
|
||||
main = bs[:-4]
|
||||
checksum = bs[-4:]
|
||||
h1 = hashlib.sha256(main).digest()
|
||||
h2 = hashlib.sha256(h1).digest()
|
||||
return h2[:4] == checksum
|
||||
|
|
|
|||
|
|
@ -87,7 +87,11 @@ class Site(object):
|
|||
# Load site settings from data/sites.json
|
||||
def loadSettings(self, settings=None):
|
||||
if not settings:
|
||||
settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address)
|
||||
try:
|
||||
settings = json.load(open(f'{config.data_dir}/sites.json')).get(self.address)
|
||||
except Exception as err:
|
||||
logging.error(f'Error loading {config.data_dir}/sites.json: {err}')
|
||||
settings = {}
|
||||
if settings:
|
||||
self.settings = settings
|
||||
if "cache" not in settings:
|
||||
|
|
|
|||
|
|
@ -202,7 +202,7 @@ class SiteAnnouncer(object):
|
|||
else:
|
||||
raise AnnounceError("Unknown protocol: %s" % address_parts["protocol"])
|
||||
except Exception as err:
|
||||
self.site.log.warning("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode))
|
||||
self.site.log.debug("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode))
|
||||
error = err
|
||||
|
||||
if error:
|
||||
|
|
|
|||
|
|
@ -31,17 +31,18 @@ class SiteManager(object):
|
|||
@util.Noparallel()
|
||||
def load(self, cleanup=True, startup=False):
|
||||
from .Site import Site
|
||||
self.log.info("Loading sites... (cleanup: %s, startup: %s)" % (cleanup, startup))
|
||||
self.log.info(f'Loading sites... ({cleanup=}, {startup=})')
|
||||
self.loaded = False
|
||||
address_found = []
|
||||
added = 0
|
||||
load_s = time.time()
|
||||
# Load new adresses
|
||||
try:
|
||||
json_path = "%s/sites.json" % config.data_dir
|
||||
json_path = f"{config.data_dir}/sites.json"
|
||||
data = json.load(open(json_path))
|
||||
except Exception as err:
|
||||
raise Exception("Unable to load %s: %s" % (json_path, err))
|
||||
self.log.error(f"Unable to load {json_path}: {err}")
|
||||
data = {}
|
||||
|
||||
sites_need = []
|
||||
|
||||
|
|
|
|||
78
src/main.py
78
src/main.py
|
|
@ -34,24 +34,79 @@ def load_config():
|
|||
|
||||
load_config()
|
||||
|
||||
def init_dirs():
|
||||
if not os.path.isdir(config.data_dir):
|
||||
os.mkdir(config.data_dir)
|
||||
try:
|
||||
os.chmod(config.data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
|
||||
except Exception as err:
|
||||
startupError("Can't change permission of %s: %s" % (config.data_dir, err))
|
||||
def importBundle(bundle):
|
||||
from zipfile import ZipFile
|
||||
from Crypt.CryptBitcoin import isValidAddress
|
||||
import json
|
||||
|
||||
sites_json = f"{config.data_dir}/sites.json"
|
||||
sites_json_path = f"{config.data_dir}/sites.json"
|
||||
try:
|
||||
with open(sites_json_path) as f:
|
||||
sites = json.load(f)
|
||||
except Exception as err:
|
||||
sites = {}
|
||||
|
||||
with ZipFile(bundle) as zf:
|
||||
all_files = zf.namelist()
|
||||
top_files = list(set(map(lambda f: f.split('/')[0], all_files)))
|
||||
if len(top_files) == 1 and not isValidAddress(top_files[0]):
|
||||
prefix = top_files[0]+'/'
|
||||
else:
|
||||
prefix = ''
|
||||
top_2 = list(set(filter(lambda f: len(f)>0,
|
||||
map(lambda f: f.removeprefix(prefix).split('/')[0], all_files))))
|
||||
for d in top_2:
|
||||
if isValidAddress(d):
|
||||
logging.info(f'unpack {d} into {config.data_dir}')
|
||||
for fname in filter(lambda f: f.startswith(prefix+d) and not f.endswith('/'), all_files):
|
||||
tgt = config.data_dir + '/' + fname.removeprefix(prefix)
|
||||
logging.info(f'-- {fname} --> {tgt}')
|
||||
info = zf.getinfo(fname)
|
||||
info.filename = tgt
|
||||
zf.extract(info)
|
||||
logging.info(f'add site {d}')
|
||||
sites[d] = {}
|
||||
else:
|
||||
logging.info(f'Warning: unknown file in a bundle: {prefix+d}')
|
||||
with open(sites_json_path, 'w') as f:
|
||||
json.dump(sites, f)
|
||||
|
||||
def init_dirs():
|
||||
data_dir = config.data_dir
|
||||
if not os.path.isdir(data_dir):
|
||||
os.mkdir(data_dir)
|
||||
try:
|
||||
os.chmod(data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
|
||||
except Exception as err:
|
||||
startupError(f"Can't change permission of {data_dir}: {err}")
|
||||
|
||||
# download latest bootstrap bundle
|
||||
if not config.disable_bootstrap and not config.offline:
|
||||
import requests
|
||||
from io import BytesIO
|
||||
|
||||
print(f'fetching {config.bootstrap_url}')
|
||||
response = requests.get(config.bootstrap_url)
|
||||
if response.status_code != 200:
|
||||
startupError(f"Cannot load bootstrap bundle (response status: {response.status_code})")
|
||||
url = response.text
|
||||
print(f'got {url}')
|
||||
response = requests.get(url)
|
||||
if response.status_code < 200 or response.status_code >= 300:
|
||||
startupError(f"Cannot load boostrap bundle (response status: {response.status_code})")
|
||||
importBundle(BytesIO(response.content))
|
||||
|
||||
sites_json = f"{data_dir}/sites.json"
|
||||
if not os.path.isfile(sites_json):
|
||||
with open(sites_json, "w") as f:
|
||||
f.write("{}")
|
||||
users_json = f"{config.data_dir}/users.json"
|
||||
users_json = f"{data_dir}/users.json"
|
||||
if not os.path.isfile(users_json):
|
||||
with open(users_json, "w") as f:
|
||||
f.write("{}")
|
||||
|
||||
# TODO: GET RID OF TOP-LEVEL CODE!!!
|
||||
config.initConsoleLogger()
|
||||
|
||||
try:
|
||||
init_dirs()
|
||||
|
|
@ -73,7 +128,7 @@ if config.action == "main":
|
|||
r = proc.wait()
|
||||
sys.exit(r)
|
||||
|
||||
config.initLogging()
|
||||
config.initLogging(console_logging=False)
|
||||
|
||||
# Debug dependent configuration
|
||||
from Debug import DebugHook
|
||||
|
|
@ -415,6 +470,9 @@ class Actions(object):
|
|||
else:
|
||||
return res
|
||||
|
||||
def importBundle(self, bundle):
|
||||
importBundle(bundle)
|
||||
|
||||
def getWebsocket(self, site):
|
||||
import websocket
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue